From 53f2841ce2fe2ec1eb6e47ff84cda9b1bbc38d34 Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 22:07:56 +0800 Subject: [PATCH 001/156] feat(chat): add the chat fact space and the interceptors three protocols share MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first commit of the chat migration. Four source protocols, ten translation directions between them, and one array of stages every chain runs — this is the space they run over and the first three entries of that array. The key layout says two things the context object could not. Each protocol has its own request and response key, so a translation consumes the source and provides the target — and `compose` then refuses a chain that could re-enter its own protocol, which is `ctx.targetApi === ` made structural. And `ingress.*` is what the client asked for, so `wantsStream` is four disjoint keys rather than one: a translated request must not inherit the target protocol's answer to a question the client never asked. That guard turns out to be two things wearing one expression. "Do not run on a re-entered request" is now structural and no stage asks it. "Which protocol was chosen" stays a live question, because a vendor normalizer genuinely needs to know — it becomes a declared fact read through `needs`, which is visibility rather than removal. So the five guards are not deleted by the migration; they change what they read. The rewrites themselves are transcribed, not redesigned: they already worked. What is new is that a stage returns the record it hands on instead of assigning to `ctx.payload`. Today's code already writes its map conditionally, which is the convention that keeps a 49-message conversation costing three objects, so the tests assert identity rather than equality — a payload no flag touched comes back as the same object, and a flag that fires but changes nothing changes nothing. Gemini is source-only. Nothing in `packages/translate/src/` targets it, so by the ruling it contributes one pipeline rather than two: a source role and no target role. --- .../data-plane/chat-interceptors_test.ts | 142 ++++++++++++++ packages/gateway/src/data-plane/chat/facts.ts | 68 +++++++ .../src/data-plane/chat/interceptors.ts | 173 ++++++++++++++++++ 3 files changed, 383 insertions(+) create mode 100644 packages/gateway/__tests__/data-plane/chat-interceptors_test.ts create mode 100644 packages/gateway/src/data-plane/chat/facts.ts create mode 100644 packages/gateway/src/data-plane/chat/interceptors.ts diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts new file mode 100644 index 0000000000..ceb94796d7 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -0,0 +1,142 @@ +// The three interceptors three chat protocols share, migrated. Two properties matter and +// neither is about the rewrites themselves — those are transcribed from code that already +// worked. What is new is that a stage returns a record instead of assigning to a context, +// and that a rewrite which changes nothing changes nothing by identity. + +import { describe, expect, it } from 'vitest'; + +import type { Chat } from '../../src/data-plane/chat/facts.ts'; +import { + applyRoleCompatibilityToChatCompletions, + disableReasoningOnForcedToolChoiceForChatCompletions, + stripPromptCacheKeyForChatCompletions, +} from '../../src/data-plane/chat/interceptors.ts'; +import { move, run, compose, defineStage } from '@floway-dev/pipeline'; +import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; +import type { ModelCandidate } from '@floway-dev/provider'; + +/** Enough of a candidate for a stage that only reads its flags. `providerModelOf` expands + * the row for the candidate's own upstream, so the shape has to be the real one. */ +const candidateWith = (...flags: string[]): ModelCandidate => ({ + model: { id: 'm', endpoints: {}, providerModels: { u: { enabledFlags: new Set(flags), pricing: null } } }, + provider: { upstreamId: 'u' }, +} as unknown as ModelCandidate); + +const payload = (messages: ChatCompletionsPayload['messages'], rest: Partial = {}): ChatCompletionsPayload => + ({ model: 'm', messages, ...rest }) as ChatCompletionsPayload; + +/** Runs one stage and hands back what it passed down, which is the only thing these + * stages do — none of them touches the response direction. The sink has to provide the + * answer the stage above declared it needs, or `compose` refuses the array; that refusal + * is the assembly check doing its job and is why the harness looks like this. */ +const handedDown = async (stage: unknown, facts: object): Promise> => { + let seen: Record = {}; + const sink = defineStage>({ + name: 'sink', + return: { provides: ['response.chat.chatCompletions'] }, + execute: async received => { + seen = received as Record; + return move({ ...received, 'response.chat.chatCompletions': { kind: 'value' as const, body: null } }); + }, + }); + await run(compose('one', [stage as never, sink]), move(facts) as never, {}); + return seen; +}; + +describe('the shared chat interceptors, as stages', () => { + it('rewrites the roles its flags name, in the settled order', async () => { + const down = await handedDown(applyRoleCompatibilityToChatCompletions, { + 'request.chat.chatCompletions': payload([ + { role: 'system', content: 'lead' }, + { role: 'user', content: 'hi' }, + { role: 'system', content: 'mid' }, + ]), + 'route.candidate': candidateWith('rewrite-system-to-developer'), + }); + const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; + expect(after.messages.map(m => m.role)).toEqual(['developer', 'user', 'developer']); + }); + + it('turns only a mid-conversation system message into a user one', async () => { + const down = await handedDown(applyRoleCompatibilityToChatCompletions, { + 'request.chat.chatCompletions': payload([ + { role: 'system', content: 'lead' }, + { role: 'system', content: 'still lead' }, + { role: 'user', content: 'hi' }, + { role: 'system', content: 'mid' }, + ]), + 'route.candidate': candidateWith('rewrite-mid-conv-system-to-user'), + }); + const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; + expect(after.messages.map(m => m.role)).toEqual(['system', 'system', 'user', 'user']); + }); + + // The convention the whole exact-recording story rests on. A stage whose flags do not + // fire must hand the payload back as the *same object*, or every layer costs a copy of + // the conversation and the dump stops being affordable. + it('hands the same payload back by identity when no flag fires', async () => { + const original = payload([{ role: 'system', content: 'lead' }]); + const down = await handedDown(applyRoleCompatibilityToChatCompletions, { + 'request.chat.chatCompletions': original, + 'route.candidate': candidateWith(), + }); + expect(down['request.chat.chatCompletions']).toBe(original); + }); + + it('hands the same payload back when a flag fires but changes nothing', async () => { + const original = payload([{ role: 'user', content: 'hi' }]); + const down = await handedDown(applyRoleCompatibilityToChatCompletions, { + 'request.chat.chatCompletions': original, + 'route.candidate': candidateWith('rewrite-system-to-developer'), + }); + expect(down['request.chat.chatCompletions']).toBe(original); + }); + + it('shares every message it did not rewrite', async () => { + const messages = Array.from({ length: 49 }, (_, i) => ( + i === 0 ? { role: 'system' as const, content: 'lead' } : { role: 'user' as const, content: `turn ${i}` } + )); + const original = payload(messages); + const down = await handedDown(applyRoleCompatibilityToChatCompletions, { + 'request.chat.chatCompletions': original, + 'route.candidate': candidateWith('rewrite-system-to-developer'), + }); + const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; + const shared = after.messages.filter((message, index) => message === messages[index]).length; + expect(shared).toBe(48); // every message but the one that changed + }); + + it('disables reasoning only when a tool choice is forced', async () => { + const forced = await handedDown(disableReasoningOnForcedToolChoiceForChatCompletions, { + 'request.chat.chatCompletions': payload([], { tool_choice: 'required' }), + 'route.candidate': candidateWith('disable-reasoning-on-forced-tool-choice'), + }); + expect((forced['request.chat.chatCompletions'] as ChatCompletionsPayload).reasoning_effort).toBe('none'); + + const free = payload([], { tool_choice: 'auto' }); + const untouched = await handedDown(disableReasoningOnForcedToolChoiceForChatCompletions, { + 'request.chat.chatCompletions': free, + 'route.candidate': candidateWith('disable-reasoning-on-forced-tool-choice'), + }); + expect(untouched['request.chat.chatCompletions']).toBe(free); + }); + + it('removes the prompt cache key, and the key itself rather than its value', async () => { + const down = await handedDown(stripPromptCacheKeyForChatCompletions, { + 'request.chat.chatCompletions': payload([], { prompt_cache_key: 'k' } as Partial), + 'route.candidate': candidateWith('strip-prompt-cache-key'), + }); + const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; + // `undefined` is not a removal: the key has to be gone, not present and empty. + expect('prompt_cache_key' in after).toBe(false); + }); + + it('leaves a payload that never had the key alone, by identity', async () => { + const original = payload([]); + const down = await handedDown(stripPromptCacheKeyForChatCompletions, { + 'request.chat.chatCompletions': original, + 'route.candidate': candidateWith('strip-prompt-cache-key'), + }); + expect(down['request.chat.chatCompletions']).toBe(original); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts new file mode 100644 index 0000000000..854d5a3f80 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -0,0 +1,68 @@ +// The chat family's fact space. Four source protocols, ten translation directions between +// them, and one shared array of stages that every chain runs. +// +// Two things the key layout says that the old context object could not. Each protocol has +// its own request key and its own response key, so a translation *consumes* the source and +// *provides* the target — and `compose` then refuses a chain that could re-enter its own +// protocol, which is `ctx.targetApi === ` deleted and made structural. And +// `ingress.*` is what the client asked for: it survives the switch to whatever protocol the +// upstream turned out to speak, which is why `wantsStream` is four disjoint keys rather +// than one shared one. + +import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; +import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; +import type { GeminiPayload } from '@floway-dev/protocols/gemini'; +import type { MessagesPayload } from '@floway-dev/protocols/messages'; +import type { ResponsesPayload } from '@floway-dev/protocols/responses'; + +/** The four protocols a client can speak here. Gemini is source-only: nothing translates + * into it, so by the ruling it contributes one pipeline rather than two — a source role + * and no target role. */ +export type ChatSourceProtocol = 'chatCompletions' | 'messages' | 'responses' | 'gemini'; + +/** The three a translation can target. */ +export type ChatTargetProtocol = Exclude; + +export interface ChatFacts extends GatewayFacts { + /** Which protocol the client spoke. Read by the stages that must know — a vendor + * normalizer genuinely does — rather than reached for through an ambient field. + * + * This is one half of what `ctx.targetApi === ` used to test. The other half, + * "do not run on a re-entered request", is not a question a stage asks any more: the + * translating stage consumed the source key, so a stage that needs it cannot be placed + * below, and `compose` says so at assembly. */ + 'ingress.chat.sourceProtocol': ChatSourceProtocol; + /** What the client asked for, per protocol and disjoint, because a translated request + * must not inherit the target protocol's answer to a question the client never asked. */ + 'ingress.chat.chatCompletions.wantsStream': boolean; + 'ingress.chat.messages.wantsStream': boolean; + 'ingress.chat.responses.wantsStream': boolean; + 'ingress.chat.gemini.wantsStream': boolean; + /** Whether the client wants the usage chunk it would otherwise never see. The gateway + * always asks the upstream for one; this decides who is shown it. */ + 'ingress.chat.chatCompletions.wantsUsageChunk': boolean; + + /** One key per protocol. A translation consumes one and provides another, which is the + * whole of what a protocol handoff is. */ + 'request.chat.chatCompletions': ChatCompletionsPayload; + 'request.chat.messages': MessagesPayload; + 'request.chat.responses': ResponsesPayload; + 'request.chat.gemini': GeminiPayload; + + /** A stream, a value and a failure sit at one key, so telling them apart is reading a + * value and no declaration ever mentions which arm is there. */ + 'response.chat.chatCompletions': ChatAnswer; + 'response.chat.messages': ChatAnswer; + 'response.chat.responses': ChatAnswer; + 'response.chat.gemini': ChatAnswer; +} + +/** What comes back at a protocol's response key. The frame type is the protocol's own; the + * three arms are the same shape for every protocol, which is why no stage dispatches on + * the arm in its declaration. */ +export type ChatAnswer = + | { readonly kind: 'stream'; readonly frames: AsyncIterable } + | { readonly kind: 'value'; readonly body: unknown } + | Failure; + +export type Chat = { [P in K]: ChatFacts[P] }; diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts new file mode 100644 index 0000000000..b53f4cbcc1 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -0,0 +1,173 @@ +// The interceptors three of the four chat protocols share, as stages. Each is written +// against one protocol's payload key and dropped into that protocol's array — one stage +// per (interceptor, protocol) pair, because the payload types differ, but one *rule* each, +// written once here. +// +// What changes from the interceptor form is the two things the pipeline replaced. A stage +// returns the record it hands on instead of assigning to `ctx.payload`, so a rewrite is a +// spread and structural sharing is what makes it cheap — today's code already writes the +// map conditionally, which is the convention that keeps a 49-message conversation costing +// three objects. And `ctx.targetApi !== ` is gone: array position says which chain a +// stage is in, and the translating stage above consumed the source key, so a re-entered +// request cannot reach a stage that needs it. + +import type { Chat } from './facts.ts'; +import { defineStage, move, transform } from '@floway-dev/pipeline'; +import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; +import { providerModelOf } from '@floway-dev/provider'; +import type { ModelCandidate } from '@floway-dev/provider'; + +/** Role rewrites, in the fixed order `system → developer → system → user`. Later rewrites + * are authoritative when flags overlap, and the last step affects only a system message + * that appears after the leading run — which is what "mid-conversation" means and why the + * fold carries a flag rather than testing the index. */ +export const applyRoleCompatibilityToChatCompletions = defineStage< + Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'applyRoleCompatibility', + through: { + request: { + needs: ['request.chat.chatCompletions', 'route.candidate'], + consumes: [], + // Declared even though it will not always be written: a stage that only modifies a + // field need not write on every run, and `provides \ needs` is the set that must be. + provides: ['request.chat.chatCompletions'], + }, + response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => ({ + request: facts => { + const rewrite = rolesFor(facts['route.candidate']); + if (rewrite === null) return facts; // the free do-nothing path + const payload = facts['request.chat.chatCompletions']; + const messages = rewriteRoles(payload.messages, rewrite); + // Written conditionally the whole way down, so a conversation this does not touch + // comes back by identity and the layer costs what the rewrite actually changed. + return messages === payload.messages + ? facts + : { ...facts, 'request.chat.chatCompletions': move({ ...payload, messages }) }; + }, + })), +}); + +interface RoleRewrite { + readonly systemToDeveloper: boolean; + readonly developerToSystem: boolean; + readonly midConversationSystemToUser: boolean; +} + +/** A flag is data on an upstream model, and this is a stage reading it. There is no + * category of stage a flag switches on or off. */ +const rolesFor = (candidate: ModelCandidate): RoleRewrite | null => { + const flags = providerModelOf(candidate).enabledFlags; + const rewrite: RoleRewrite = { + systemToDeveloper: flags.has('rewrite-system-to-developer'), + developerToSystem: flags.has('rewrite-developer-to-system'), + midConversationSystemToUser: flags.has('rewrite-mid-conv-system-to-user'), + }; + return rewrite.systemToDeveloper || rewrite.developerToSystem || rewrite.midConversationSystemToUser + ? rewrite + : null; +}; + +type Messages = ChatCompletionsPayload['messages']; + +const rewriteRoles = (messages: Messages, rewrite: RoleRewrite): Messages => { + let crossedLeadingSystemRun = false; + let changed = false; + const mapped = messages.map(message => { + let result = message; + if (rewrite.systemToDeveloper && result.role === 'system') result = { ...result, role: 'developer' }; + if (rewrite.developerToSystem && result.role === 'developer') result = { ...result, role: 'system' }; + if (!crossedLeadingSystemRun && result.role !== 'system') crossedLeadingSystemRun = true; + if (rewrite.midConversationSystemToUser && crossedLeadingSystemRun && result.role === 'system') { + result = { ...result, role: 'user' }; + } + if (result !== message) changed = true; + return result; + }); + return changed ? mapped : messages; +}; + +/** The reasoning sentinel a forced tool choice needs. Gated by its flag, and the sentinel + * is the gateway's canonical form — putting it on the wire in a vendor's shape is the + * vendor normalizer's job, which is why this runs above them. */ +export const disableReasoningOnForcedToolChoiceForChatCompletions = defineStage< + Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'disableReasoningOnForcedToolChoice', + through: { + request: { + needs: ['request.chat.chatCompletions', 'route.candidate'], + consumes: [], + provides: ['request.chat.chatCompletions'], + }, + response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => ({ + request: facts => { + if (!providerModelOf(facts['route.candidate']).enabledFlags.has('disable-reasoning-on-forced-tool-choice')) { + return facts; + } + const payload = facts['request.chat.chatCompletions']; + if (!isForcedToolChoice(payload.tool_choice)) return facts; + return { ...facts, 'request.chat.chatCompletions': move({ ...payload, reasoning_effort: 'none' }) }; + }, + })), +}); + +/** `required`, or a named function. `auto` and `none` leave the model free to reason. */ +const isForcedToolChoice = (choice: ChatCompletionsPayload['tool_choice']): boolean => + choice === 'required' || (typeof choice === 'object' && choice !== null); + +/** Drops a field the upstream would reject as an unknown argument. It runs above the vendor + * normalizers so each of them sees the already-stripped canonical payload. */ +export const stripPromptCacheKeyForChatCompletions = defineStage< + Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'stripPromptCacheKey', + through: { + request: { + needs: ['request.chat.chatCompletions', 'route.candidate'], + consumes: [], + provides: ['request.chat.chatCompletions'], + }, + response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => ({ + request: facts => { + if (!providerModelOf(facts['route.candidate']).enabledFlags.has('strip-prompt-cache-key')) return facts; + const payload = facts['request.chat.chatCompletions']; + if (!('prompt_cache_key' in payload)) return facts; + // Removing a field inside a value is the same move as removing a fact: name it in the + // destructuring and spread the rest. + const { prompt_cache_key: dropped, ...rest } = payload; + void dropped; + return { ...facts, 'request.chat.chatCompletions': move(rest) }; + }, + })), +}); From 5a19c6d01c4a2d39116b404772ea716666ca5aa2 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:37:48 +0800 Subject: [PATCH 002/156] fix(chat): read an attempt's flags from the selector that carries them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chat interceptors were written against route.candidate, the key the selector split replaced. They reach for the live candidate only to read its enabled flags, and the selector carries those as data — so they now read what travels in the record and the tests need no candidate stub at all. The shared chat stages arrive with them: resolveChatCandidates enumerates, narrows to the wires a source protocol can reach, and orders by affinity, which can refuse outright when a turn's own state requires two upstreams at once. --- .../data-plane/chat-interceptors_test.ts | 29 ++--- .../src/data-plane/chat/interceptors.ts | 35 +++--- .../gateway/src/data-plane/chat/stages.ts | 118 ++++++++++++++++++ 3 files changed, 147 insertions(+), 35 deletions(-) create mode 100644 packages/gateway/src/data-plane/chat/stages.ts diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index ceb94796d7..a4a31f62a4 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -11,16 +11,13 @@ import { disableReasoningOnForcedToolChoiceForChatCompletions, stripPromptCacheKeyForChatCompletions, } from '../../src/data-plane/chat/interceptors.ts'; +import type { AttemptSelector } from '../../src/data-plane/pipeline/facts.ts'; import { move, run, compose, defineStage } from '@floway-dev/pipeline'; import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import type { ModelCandidate } from '@floway-dev/provider'; -/** Enough of a candidate for a stage that only reads its flags. `providerModelOf` expands - * the row for the candidate's own upstream, so the shape has to be the real one. */ -const candidateWith = (...flags: string[]): ModelCandidate => ({ - model: { id: 'm', endpoints: {}, providerModels: { u: { enabledFlags: new Set(flags), pricing: null } } }, - provider: { upstreamId: 'u' }, -} as unknown as ModelCandidate); +/** What an attempt is, to a stage that only reads flags. The selector carries them as data, + * so nothing here needs the live candidate the resolver kept. */ +const attemptWith = (...flags: string[]): AttemptSelector => ({ upstreamId: 'u', modelId: 'm', flags }); const payload = (messages: ChatCompletionsPayload['messages'], rest: Partial = {}): ChatCompletionsPayload => ({ model: 'm', messages, ...rest }) as ChatCompletionsPayload; @@ -51,7 +48,7 @@ describe('the shared chat interceptors, as stages', () => { { role: 'user', content: 'hi' }, { role: 'system', content: 'mid' }, ]), - 'route.candidate': candidateWith('rewrite-system-to-developer'), + 'route.attempt': attemptWith('rewrite-system-to-developer'), }); const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; expect(after.messages.map(m => m.role)).toEqual(['developer', 'user', 'developer']); @@ -65,7 +62,7 @@ describe('the shared chat interceptors, as stages', () => { { role: 'user', content: 'hi' }, { role: 'system', content: 'mid' }, ]), - 'route.candidate': candidateWith('rewrite-mid-conv-system-to-user'), + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), }); const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; expect(after.messages.map(m => m.role)).toEqual(['system', 'system', 'user', 'user']); @@ -78,7 +75,7 @@ describe('the shared chat interceptors, as stages', () => { const original = payload([{ role: 'system', content: 'lead' }]); const down = await handedDown(applyRoleCompatibilityToChatCompletions, { 'request.chat.chatCompletions': original, - 'route.candidate': candidateWith(), + 'route.attempt': attemptWith(), }); expect(down['request.chat.chatCompletions']).toBe(original); }); @@ -87,7 +84,7 @@ describe('the shared chat interceptors, as stages', () => { const original = payload([{ role: 'user', content: 'hi' }]); const down = await handedDown(applyRoleCompatibilityToChatCompletions, { 'request.chat.chatCompletions': original, - 'route.candidate': candidateWith('rewrite-system-to-developer'), + 'route.attempt': attemptWith('rewrite-system-to-developer'), }); expect(down['request.chat.chatCompletions']).toBe(original); }); @@ -99,7 +96,7 @@ describe('the shared chat interceptors, as stages', () => { const original = payload(messages); const down = await handedDown(applyRoleCompatibilityToChatCompletions, { 'request.chat.chatCompletions': original, - 'route.candidate': candidateWith('rewrite-system-to-developer'), + 'route.attempt': attemptWith('rewrite-system-to-developer'), }); const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; const shared = after.messages.filter((message, index) => message === messages[index]).length; @@ -109,14 +106,14 @@ describe('the shared chat interceptors, as stages', () => { it('disables reasoning only when a tool choice is forced', async () => { const forced = await handedDown(disableReasoningOnForcedToolChoiceForChatCompletions, { 'request.chat.chatCompletions': payload([], { tool_choice: 'required' }), - 'route.candidate': candidateWith('disable-reasoning-on-forced-tool-choice'), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); expect((forced['request.chat.chatCompletions'] as ChatCompletionsPayload).reasoning_effort).toBe('none'); const free = payload([], { tool_choice: 'auto' }); const untouched = await handedDown(disableReasoningOnForcedToolChoiceForChatCompletions, { 'request.chat.chatCompletions': free, - 'route.candidate': candidateWith('disable-reasoning-on-forced-tool-choice'), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); expect(untouched['request.chat.chatCompletions']).toBe(free); }); @@ -124,7 +121,7 @@ describe('the shared chat interceptors, as stages', () => { it('removes the prompt cache key, and the key itself rather than its value', async () => { const down = await handedDown(stripPromptCacheKeyForChatCompletions, { 'request.chat.chatCompletions': payload([], { prompt_cache_key: 'k' } as Partial), - 'route.candidate': candidateWith('strip-prompt-cache-key'), + 'route.attempt': attemptWith('strip-prompt-cache-key'), }); const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; // `undefined` is not a removal: the key has to be gone, not present and empty. @@ -135,7 +132,7 @@ describe('the shared chat interceptors, as stages', () => { const original = payload([]); const down = await handedDown(stripPromptCacheKeyForChatCompletions, { 'request.chat.chatCompletions': original, - 'route.candidate': candidateWith('strip-prompt-cache-key'), + 'route.attempt': attemptWith('strip-prompt-cache-key'), }); expect(down['request.chat.chatCompletions']).toBe(original); }); diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index b53f4cbcc1..7b1ab104ff 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -14,15 +14,13 @@ import type { Chat } from './facts.ts'; import { defineStage, move, transform } from '@floway-dev/pipeline'; import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; -import type { ModelCandidate } from '@floway-dev/provider'; /** Role rewrites, in the fixed order `system → developer → system → user`. Later rewrites * are authoritative when flags overlap, and the last step affects only a system message * that appears after the leading run — which is what "mid-conversation" means and why the * fold carries a flag rather than testing the index. */ export const applyRoleCompatibilityToChatCompletions = defineStage< - Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions' | 'route.attempt'>, Chat<'request.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'> @@ -30,7 +28,7 @@ export const applyRoleCompatibilityToChatCompletions = defineStage< name: 'applyRoleCompatibility', through: { request: { - needs: ['request.chat.chatCompletions', 'route.candidate'], + needs: ['request.chat.chatCompletions', 'route.attempt'], consumes: [], // Declared even though it will not always be written: a stage that only modifies a // field need not write on every run, and `provides \ needs` is the set that must be. @@ -39,13 +37,13 @@ export const applyRoleCompatibilityToChatCompletions = defineStage< response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions' | 'route.attempt'>, Chat<'request.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'> >(() => ({ request: facts => { - const rewrite = rolesFor(facts['route.candidate']); + const rewrite = rolesFor(facts['route.attempt'].flags); if (rewrite === null) return facts; // the free do-nothing path const payload = facts['request.chat.chatCompletions']; const messages = rewriteRoles(payload.messages, rewrite); @@ -66,12 +64,11 @@ interface RoleRewrite { /** A flag is data on an upstream model, and this is a stage reading it. There is no * category of stage a flag switches on or off. */ -const rolesFor = (candidate: ModelCandidate): RoleRewrite | null => { - const flags = providerModelOf(candidate).enabledFlags; +const rolesFor = (flags: readonly string[]): RoleRewrite | null => { const rewrite: RoleRewrite = { - systemToDeveloper: flags.has('rewrite-system-to-developer'), - developerToSystem: flags.has('rewrite-developer-to-system'), - midConversationSystemToUser: flags.has('rewrite-mid-conv-system-to-user'), + systemToDeveloper: flags.includes('rewrite-system-to-developer'), + developerToSystem: flags.includes('rewrite-developer-to-system'), + midConversationSystemToUser: flags.includes('rewrite-mid-conv-system-to-user'), }; return rewrite.systemToDeveloper || rewrite.developerToSystem || rewrite.midConversationSystemToUser ? rewrite @@ -101,7 +98,7 @@ const rewriteRoles = (messages: Messages, rewrite: RoleRewrite): Messages => { * is the gateway's canonical form — putting it on the wire in a vendor's shape is the * vendor normalizer's job, which is why this runs above them. */ export const disableReasoningOnForcedToolChoiceForChatCompletions = defineStage< - Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions' | 'route.attempt'>, Chat<'request.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'> @@ -109,20 +106,20 @@ export const disableReasoningOnForcedToolChoiceForChatCompletions = defineStage< name: 'disableReasoningOnForcedToolChoice', through: { request: { - needs: ['request.chat.chatCompletions', 'route.candidate'], + needs: ['request.chat.chatCompletions', 'route.attempt'], consumes: [], provides: ['request.chat.chatCompletions'], }, response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions' | 'route.attempt'>, Chat<'request.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'> >(() => ({ request: facts => { - if (!providerModelOf(facts['route.candidate']).enabledFlags.has('disable-reasoning-on-forced-tool-choice')) { + if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { return facts; } const payload = facts['request.chat.chatCompletions']; @@ -139,7 +136,7 @@ const isForcedToolChoice = (choice: ChatCompletionsPayload['tool_choice']): bool /** Drops a field the upstream would reject as an unknown argument. It runs above the vendor * normalizers so each of them sees the already-stripped canonical payload. */ export const stripPromptCacheKeyForChatCompletions = defineStage< - Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions' | 'route.attempt'>, Chat<'request.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'> @@ -147,20 +144,20 @@ export const stripPromptCacheKeyForChatCompletions = defineStage< name: 'stripPromptCacheKey', through: { request: { - needs: ['request.chat.chatCompletions', 'route.candidate'], + needs: ['request.chat.chatCompletions', 'route.attempt'], consumes: [], provides: ['request.chat.chatCompletions'], }, response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.chatCompletions' | 'route.candidate'>, + Chat<'request.chat.chatCompletions' | 'route.attempt'>, Chat<'request.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'>, Chat<'response.chat.chatCompletions'> >(() => ({ request: facts => { - if (!providerModelOf(facts['route.candidate']).enabledFlags.has('strip-prompt-cache-key')) return facts; + if (!facts['route.attempt'].flags.includes('strip-prompt-cache-key')) return facts; const payload = facts['request.chat.chatCompletions']; if (!('prompt_cache_key' in payload)) return facts; // Removing a field inside a value is the same move as removing a fact: name it in the diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts new file mode 100644 index 0000000000..d44a497427 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -0,0 +1,118 @@ +// What every chat chain runs, and what it takes from the family that runs it. +// +// Chat differs from the families in PR 2 in two ways, and both are here rather than in each +// family. Candidates are ordered by affinity before they are tried, because client-carried +// state pins a turn to the upstream that issued it. And the last stage picks a *wire*: a +// candidate that serves this request may speak a different protocol than the client did, so +// what follows the fork is decided per candidate rather than at assembly. +// +// The second is why dispatch is an `into` stage. Only a pipeline's last stage may name what +// comes next, and failover re-runs the whole suffix — including that last stage — so the +// next candidate re-picks its own wire with no mechanism of its own. + +import type { ChatSourceProtocol } from './facts.ts'; +import type { AttemptSelector, GatewayFacts } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { enumerateModelCandidates } from '../providers/resolution.ts'; +import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; +import { selectAffinityCandidates } from './shared/affinity/index.ts'; +import type { AffinityRequestAnalysis } from './shared/affinity/selection.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { providerModelOf, type ModelCandidate } from '@floway-dev/provider'; + +type Slice = { [P in K]: GatewayFacts[P] }; + +/** What a chat family needs beyond the shared services: the affinity selection this run + * made, so the stage that dials can ask for the payload the winning candidate is owed. */ +export interface ChatServices extends GatewayServices { + readonly rememberChatSelection: (payloadFor: (candidate: ModelCandidate) => unknown) => void; + readonly chatPayloadFor: (selector: AttemptSelector) => unknown; + readonly selectAffinity: (candidate: ModelCandidate) => void; +} + +/** How a source protocol narrows and orders the candidates that could serve it. */ +export interface ChatNarrowing { + readonly source: ChatSourceProtocol; + /** Which upstream wires this source prefers, in order. A candidate none of whose endpoints + * appear here cannot serve the request whatever else it offers. */ + readonly canServe: (candidate: ModelCandidate) => boolean; + /** What the client's own turn says about where it must go. Client-carried state — a + * Responses `previous_response_id`, an encrypted reasoning blob — pins the turn to the + * upstream that issued it, so this runs before any candidate is tried. */ + readonly affinity: () => Promise>; + readonly unsupported: (model: string) => string; + readonly refuse: (status: number, message: string) => Refusal; + readonly refuses: readonly (keyof Refusal)[]; +} + +/** + * Provides `serve.candidates` in the order affinity asks for, or answers with the failure + * that says why there are none. + * + * Affinity can refuse outright — a turn whose state requires two incompatible upstreams has + * nowhere to go — and that is an answer this stage already holds, which is why it carries + * the `return` trait alongside `through`. + */ +export const resolveChatCandidates = (narrowing: ChatNarrowing) => defineStage< + Slice<'serve.model'>, + Slice<'serve.model' | 'serve.candidates'>, + Slice<'response.usage.billable' | 'response.http.headers'>, + Slice<'response.usage.billable' | 'response.http.headers'>, + Slice<'response.usage.billable' | 'response.http.headers'> & Refusal, + ChatServices +>({ + name: 'resolveChatCandidates', + through: { + request: { needs: ['serve.model'], consumes: [], provides: ['serve.candidates'] }, + response: { needs: ['response.usage.billable', 'response.http.headers'], consumes: [], provides: [] }, + }, + return: { provides: ['response.usage.billable', 'response.http.headers', ...narrowing.refuses] }, + execute: async (facts, next, use) => { + const model = facts['serve.model']; + const affinity = await narrowing.affinity(); + const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ + upstreamIds: use.gateway.upstreamIds, + model, + kind: 'chat', + scheduler: use.gateway.backgroundScheduler, + runtimeLocation: use.gateway.runtimeLocation, + }); + + // An empty billed set is what "we did not call an upstream" looks like, and an empty + // header list is the same statement on the other key. + const refuse = (status: number, message: string) => + move({ + ...facts, + 'response.usage.billable': [], + 'response.http.headers': [], + ...narrowing.refuse(status, message), + }); + + const viable = candidates.filter(candidate => narrowing.canServe(candidate)); + const selection = selectAffinityCandidates(viable, affinity); + if ('kind' in selection) return refuse(409, selection.message); + if (selection.candidates.length === 0) { + const missing = sawModel + ? narrowing.unsupported(model) + : `Model ${model} is not available on any configured upstream.`; + return refuse(sawModel ? 400 : 404, appendFailedUpstreams(missing, failedUpstreams)); + } + + // The live half stays with the resolver; only selectors travel. The payload a candidate + // is owed is part of that live half — affinity materializes it per candidate, and it + // carries the client's own state rewritten for the upstream that will see it. + use.rememberCandidates(selection.candidates); + use.rememberChatSelection(selection.payloadFor); + use.log.debug('resolved chat candidates', { model, viable: selection.candidates.length, resolved: candidates.length }); + return await next({ ...facts, 'serve.candidates': move(selection.candidates.map(selectorFor)) }); + }, +}); + +/** Everything about a candidate that is data. The live half — the provider instance, the + * fetcher, the models cache — stays out of the record and is looked back up by the + * resolver service at the moment of the call. */ +const selectorFor = (candidate: ModelCandidate): AttemptSelector => ({ + upstreamId: candidate.provider.upstreamId, + modelId: candidate.model.id, + flags: [...providerModelOf(candidate).enabledFlags], +}); From fa7e65cae9120f611bf8fab479b4b99490d1bd7d Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:46:31 +0800 Subject: [PATCH 003/156] feat(chat): assemble the Chat Completions chain on the pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reference chat chain, and what it establishes for the three families that follow: the edge decides whether the client sees the frames or the one object they add up to, which is where the stream-to-value collection belongs — the upstream speaks SSE whatever the client asked for, so folding is the edge's own work rather than a second reading. The shared chat resolver narrows to the wires a source protocol can reach and orders what is left by affinity, which can refuse a turn whose carried state needs two upstreams at once. The interceptors run as ordinary stages between the fork and the ending. Metering reads the upstream's own usage off its own events as they pass, so a streaming turn settles from the promise its last chunk resolves rather than in the stage. The route still runs the existing surface. --- .../data-plane/pipeline/assembly_test.ts | 2 + .../chat/chat-completions/pipeline.ts | 313 ++++++++++++++++++ .../gateway/src/data-plane/chat/stages.ts | 6 +- 3 files changed, 319 insertions(+), 2 deletions(-) create mode 100644 packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 88ff36a6d6..7131930d01 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from 'vitest'; import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/pipeline.ts'; +import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; import { imagesServePipeline } from '../../../src/data-plane/images/pipeline.ts'; @@ -21,6 +22,7 @@ const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] ['completions', () => completionsServePipeline], ['audio transcription', () => audioTranscriptionServePipeline], ['alpha search', () => searchServePipeline({ kind: 'search' } as never)], + ['chat completions', () => chatCompletionsServePipeline({ model: 'm', messages: [] } as never)], ]; describe('every family assembles', () => { diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts new file mode 100644 index 0000000000..1e293f58ac --- /dev/null +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -0,0 +1,313 @@ +// Chat Completions as a pipeline — the reference chain for the chat families. +// +// emitChatCompletions the edge: writes the answer in the shape the client asked for +// writeSettlement above the fork, so a run bills once however many wires it tried +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// the interceptors the shared array, each one a stage +// callChatCompletionsUpstream the ending: dials this candidate's wire +// +// The wire a candidate is dialled on is chosen per candidate rather than at assembly, and +// because failover re-runs the whole suffix the next candidate re-picks its own. Only the +// native wire is here; the translated ones hand up the same key, which is what will make +// them interchangeable with it. + +import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; +import { billableUsageFromChatCompletionsEvent } from './usage.ts'; +import type { UsageQuantities } from '../../../repo/types.ts'; +import type { BillableEntity } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { failover } from '../../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; +import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import type { ChatFacts } from '../facts.ts'; +import { + applyRoleCompatibilityToChatCompletions, + disableReasoningOnForcedToolChoiceForChatCompletions, + stripPromptCacheKeyForChatCompletions, +} from '../interceptors.ts'; +import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { + chatCompletionsProtocolFrameToSSEFrame, + collectChatCompletionsProtocolEventsToResult, + type ChatCompletionsPayload, + type ChatCompletionsStreamEvent, +} from '@floway-dev/protocols/chat-completions'; +import { renderErrorEnvelope, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; + +/** `/v1/chat/completions` prefers its own wire, then the translated Messages path, then the + * translated Responses path. */ +export const chatCompletionsTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); + +/** What this family adds to the chat space. */ +export interface ChatCompletionsFacts extends ChatFacts { + /** What the client is actually sent — an object when it asked for one, SSE frames when it + * asked to stream. The edge provides it, so a dump shows what the client received. */ + 'response.chat.chatCompletions.rendered': Record | AsyncIterable; + /** What the upstream will have reported once the frames run out, and `null` when nothing + * streamed. Settling from this is the epilogue's job, after the drain. */ + 'response.chat.chatCompletions.streamedUsage': Promise | null; +} + +type C = { [P in K]: ChatCompletionsFacts[P] }; + +/** + * The outermost edge. A chat answer is always a stream by the time it reaches here — the + * upstream speaks SSE whatever the client asked for — so what this decides is whether the + * client sees the frames or the one object they add up to. + * + * Collecting is therefore the edge's own work and not a second reading of the upstream: the + * same frames that would have gone out are folded here instead. + */ +const emitChatCompletions = defineStage< + C<'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk'>, + C<'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk'>, + C<'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk' + | 'response.chat.chatCompletions' | 'response.http.headers'>, + C<'response.chat.chatCompletions.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitChatCompletions', + through: { + request: { + needs: ['ingress.chat.chatCompletions.wantsStream', 'ingress.chat.chatCompletions.wantsUsageChunk'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.chat.chatCompletions', 'response.http.headers'], + consumes: ['response.chat.chatCompletions', 'response.http.headers'], + provides: ['response.chat.chatCompletions.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.chat.chatCompletions': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.chatCompletions.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.http.status': answer.status, + }; + } + if (answer.kind === 'value') { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.chatCompletions.rendered': move(answer.body as Record), + 'response.http.status': 200, + }; + } + + const frames = answer.frames as AsyncIterable>; + if (!back['ingress.chat.chatCompletions.wantsStream']) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.chatCompletions.rendered': move( + await collectChatCompletionsProtocolEventsToResult(frames) as unknown as Record, + ), + 'response.http.status': 200, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.chatCompletions.rendered': renderSSE(frames, back['ingress.chat.chatCompletions.wantsUsageChunk']), + 'response.http.status': 200, + }; + }, +}); + +/** Metering asks the upstream for a usage chunk on every streaming turn; whether the client + * is shown it is the client's own question. The protocol owns which frame that is, and says + * so by writing no frame at all. */ +const renderSSE = ( + frames: AsyncIterable>, + includeUsageChunk: boolean, +): AsyncIterable => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + const written = chatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk }); + if (written !== null) yield written; + } + })(), +}); + +/** + * The native wire. It dials Chat Completions and provides the answer at this family's own + * response key — which is what will make it interchangeable with a translated chain: both + * hand up `response.chat.chatCompletions`, and the stage above cannot tell which ran. + */ +const callChatCompletionsUpstream = defineStage< + C<'request.chat.chatCompletions' | 'route.attempt' | 'ingress.http.headers'>, + C<'response.chat.chatCompletions' | 'response.chat.chatCompletions.streamedUsage' + | 'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'callChatCompletionsUpstream', + return: { + provides: [ + 'response.chat.chatCompletions', + 'response.chat.chatCompletions.streamedUsage', + 'response.usage.billable', + 'response.http.headers', + ], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + + // The id the client addressed does not travel: the provider re-stamps whatever it + // resolved upstream, and an alias' own rules are applied to the body that is sent. + const payload = { ...facts['request.chat.chatCompletions'], model: candidate.model.id }; + if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(payload, candidate.rules); + const { model: _addressed, ...body } = payload; + + let result; + try { + result = await candidate.provider.instance.callChatCompletions( + providerModelOf(candidate), + body, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.chat.chatCompletions': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.chatCompletions.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.headers': [], + }); + } + + const identity = telemetryModelIdentity(candidate, result.modelKey); + // An upstream that was called and reported nothing, which is a different statement from + // reporting zero. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + + if (!result.ok) { + const text = await result.response.text(); + use.log.warn('upstream refused', { status: result.response.status }); + let parsed: unknown; + try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } + return move({ + ...facts, + 'response.chat.chatCompletions': { + status: result.response.status, + message: text, + ...(parsed === undefined ? {} : { body: parsed }), + }, + 'response.chat.chatCompletions.streamedUsage': null, + 'response.usage.billable': called, + 'response.http.headers': [...result.response.headers], + }); + } + + const metered = meterChatCompletions(result.events, identity); + return move({ + ...facts, + 'response.chat.chatCompletions': { kind: 'stream' as const, frames: metered.frames }, + 'response.chat.chatCompletions.streamedUsage': metered.billable, + 'response.usage.billable': called, + 'response.http.headers': [...(result.headers ?? new Headers())], + }); + }, +}); + +/** Reads the upstream's own usage off its own events as they pass, so the reading costs one + * pass and the client's stream is what drives it. Only a report carrying real counts + * replaces the running figure, so a trailing empty usage frame cannot wipe a good one. */ +const meterChatCompletions = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, +): { readonly frames: AsyncIterable>; readonly billable: Promise } => { + let settle!: (billable: readonly BillableEntity[]) => void; + const billable = new Promise(resolve => { settle = resolve; }); + const generator = (async function* () { + let reported: BillableUsage | undefined; + try { + for await (const frame of source) { + if (frame.type === 'event') { + const usage = billableUsageFromChatCompletionsEvent(frame.event); + if (usage !== null) reported = usage; + } + yield frame; + } + } finally { + // Reached however the frames ended — the terminal chunk, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + settle([{ identity, quantities: billed(reported) }]); + } + })(); + return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; +}; + +/** An upstream that reported nothing leaves no quantities at all, which is a different + * statement from reporting zero. */ +const billed = (usage: BillableUsage | undefined): UsageQuantities => + usage === undefined ? {} : tokenUsageFromBillableUsage(usage) as unknown as UsageQuantities; + +/** A candidate that cannot serve *this* request is not a candidate — and what the client's + * own turn carries decides the order the rest are tried in, which is why the narrowing is + * built from the request rather than being a constant. */ +const narrowing = (payload: ChatCompletionsPayload): ChatNarrowing> => ({ + source: 'chatCompletions', + canServe: candidate => chatCompletionsTarget.canServe(candidate.model.endpoints), + affinity: async gateway => await analyzeChatCompletionsAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the /chat/completions endpoint.`, + refuse: (status, message) => ({ 'response.chat.chatCompletions': { status, message } }), + refuses: ['response.chat.chatCompletions'], +}); + +export type ChatCompletionsServeEntry = C< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' + | 'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk' + | 'request.chat.chatCompletions' | 'serve.model' +>; + +export type ChatCompletionsServeExit = C< + 'response.chat.chatCompletions.rendered' | 'response.chat.chatCompletions.streamedUsage' + | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' +>; + +export const chatCompletionsServePipeline = (payload: ChatCompletionsPayload): Pipeline => + compose('chatCompletionsServe', [ + emitChatCompletions, + writeSettlement( + handedUp => isFailure((handedUp as { 'response.chat.chatCompletions'?: unknown })['response.chat.chatCompletions']), + handedUp => (handedUp as { 'response.chat.chatCompletions.streamedUsage'?: unknown })['response.chat.chatCompletions.streamedUsage'] !== null, + ), + resolveChatCandidates(narrowing(payload)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.chatCompletions'?: unknown })['response.chat.chatCompletions']), + owns: [], + }), + applyRoleCompatibilityToChatCompletions, + disableReasoningOnForcedToolChoiceForChatCompletions, + stripPromptCacheKeyForChatCompletions, + callChatCompletionsUpstream, + ]); diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts index d44a497427..842c454aab 100644 --- a/packages/gateway/src/data-plane/chat/stages.ts +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -17,6 +17,7 @@ import { enumerateModelCandidates } from '../providers/resolution.ts'; import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; import { selectAffinityCandidates } from './shared/affinity/index.ts'; import type { AffinityRequestAnalysis } from './shared/affinity/selection.ts'; +import type { ChatGatewayCtx } from './shared/gateway-ctx.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { providerModelOf, type ModelCandidate } from '@floway-dev/provider'; @@ -25,6 +26,7 @@ type Slice = { [P in K]: GatewayFacts[P] }; /** What a chat family needs beyond the shared services: the affinity selection this run * made, so the stage that dials can ask for the payload the winning candidate is owed. */ export interface ChatServices extends GatewayServices { + readonly gateway: ChatGatewayCtx; readonly rememberChatSelection: (payloadFor: (candidate: ModelCandidate) => unknown) => void; readonly chatPayloadFor: (selector: AttemptSelector) => unknown; readonly selectAffinity: (candidate: ModelCandidate) => void; @@ -39,7 +41,7 @@ export interface ChatNarrowing { /** What the client's own turn says about where it must go. Client-carried state — a * Responses `previous_response_id`, an encrypted reasoning blob — pins the turn to the * upstream that issued it, so this runs before any candidate is tried. */ - readonly affinity: () => Promise>; + readonly affinity: (gateway: ChatGatewayCtx) => Promise>; readonly unsupported: (model: string) => string; readonly refuse: (status: number, message: string) => Refusal; readonly refuses: readonly (keyof Refusal)[]; @@ -69,7 +71,7 @@ export const resolveChatCandidates = (narrowing: ChatNar return: { provides: ['response.usage.billable', 'response.http.headers', ...narrowing.refuses] }, execute: async (facts, next, use) => { const model = facts['serve.model']; - const affinity = await narrowing.affinity(); + const affinity = await narrowing.affinity(use.gateway); const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ upstreamIds: use.gateway.upstreamIds, model, From c892738657a228fb140a5921c6a0b21ccddcbf0d Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:50:43 +0800 Subject: [PATCH 004/156] test(chat): run the Chat Completions chain, not just assemble it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chain composed and nothing had run it. Doing so found the edge handing its SSE view up without move(), which the handover gate refuses — a stream the run never took ownership of would have escaped the record's own guarantee. The five cases are the ones only running can state: frames written out when the client asked to stream, the same frames folded into one object when it did not, the upstream's forwardable headers kept and content-length dropped, a refusal carrying the upstream's own status and words, and a dial nobody answered failing over to the next candidate rather than ending the run. --- .../chat/chatcompletions-pipeline_test.ts | 167 ++++++++++++++++++ .../chat/chat-completions/pipeline.ts | 2 +- 2 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts new file mode 100644 index 0000000000..f9ef17b78c --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -0,0 +1,167 @@ +// The Chat Completions chain, run. Assembly is checked where every family's is; what is +// written down here is what only running it can say — that the edge folds a stream into one +// object when the client did not ask to stream, that a refusal keeps the upstream's own +// status and words, and that a dial nobody answered is a value the fork can move past. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import { directFetcher, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +const candidate = ( + callChatCompletions: () => Promise>, + upstreamId = 'up_a', +): ModelCandidate => { + const endpoints = { chatCompletions: {} }; + return { + provider: { + upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callChatCompletions }), + }, + model: stubInternalModel( + { id: 'chat-model', endpoints, providerModels: { [upstreamId]: stubProviderModel({ id: 'chat-model', endpoints }) } }, + upstreamId, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const chunk = (text: string): ChatCompletionsStreamEvent => ({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [{ index: 0, delta: { content: text }, finish_reason: null }], +}); + +const stream = (...events: readonly ChatCompletionsStreamEvent[]): ProviderStreamResult => ({ + ok: true, + modelKey: 'chat-model-key', + headers: new Headers({ 'x-request-id': 'req-1', 'content-length': '99' }), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event }; + yield { type: 'done' as const }; + })(), +}); + +const payload = { model: 'chat-model', messages: [{ role: 'user', content: 'hi' }] } as unknown as ChatCompletionsPayload; + +const serve = async (wantsStream: boolean) => await run( + chatCompletionsServePipeline(payload), + move({ + 'ingress.http.headers': [] as readonly (readonly [string, string])[], + 'ingress.chat.sourceProtocol': 'chatCompletions', + 'ingress.chat.chatCompletions.wantsStream': wantsStream, + 'ingress.chat.chatCompletions.wantsUsageChunk': false, + 'request.chat.chatCompletions': payload, + 'serve.model': 'chat-model', + }) as never, + { + gateway: mockChatGatewayCtx({ wantsStream }), + background: () => {}, + rememberCandidates: () => {}, + rememberChatSelection: () => {}, + chatPayloadFor: () => undefined, + selectAffinity: () => {}, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, +); + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the chat completions chain', () => { + it('writes the frames out when the client asked to stream', async () => { + resolves([candidate(async () => stream(chunk('he'), chunk('llo')))]); + + const { facts } = await serve(true); + const frames: SseFrame[] = []; + for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); + + expect(facts['response.http.status']).toBe(200); + expect(frames.map(frame => frame.data)).toEqual([ + JSON.stringify(chunk('he')), + JSON.stringify(chunk('llo')), + '[DONE]', + ]); + }); + + // The upstream speaks SSE whatever the client asked for, so a client that did not ask to + // stream is answered from the same frames — folded here rather than read a second time. + it('folds the frames into one object when the client did not', async () => { + resolves([candidate(async () => stream(chunk('he'), chunk('llo')))]); + + const { facts } = await serve(false); + const rendered = facts['response.chat.chatCompletions.rendered'] as { choices: { message: { content: string } }[] }; + + expect(facts['response.http.status']).toBe(200); + expect(rendered.choices[0]!.message.content).toBe('hello'); + }); + + // Content-length would misdescribe a body this gateway serialized itself; a vendor trace + // is what a client and an operator both need to correlate a turn. + it('forwards the upstream headers a client may see and drops the ones it may not', async () => { + resolves([candidate(async () => stream(chunk('hi')))]); + + const { facts } = await serve(true); + + expect(Object.fromEntries(facts['response.http.headers'])).toMatchObject({ 'x-request-id': 'req-1' }); + expect(Object.fromEntries(facts['response.http.headers'])).not.toHaveProperty('content-length'); + }); + + it('answers an upstream refusal with its own status and words', async () => { + resolves([candidate(async () => ({ + ok: false, + modelKey: 'chat-model-key', + response: new Response(JSON.stringify({ error: { message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + }))]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(429); + expect(facts['response.chat.chatCompletions.rendered']).toEqual({ error: { message: 'slow down' } }); + }); + + // A refused connection is an outcome the fork has to be able to see, not a fault that ends + // the run — so the second candidate is tried and its answer is the one served. + it('fails a dial that never connected over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); }, 'up_dead'), + candidate(async () => { tried.push('alive'); return stream(chunk('hi')); }, 'up_alive'), + ]); + + const { facts } = await serve(false); + + expect(tried).toEqual(['dead', 'alive']); + expect(facts['response.http.status']).toBe(200); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 1e293f58ac..4e5b06acdd 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -126,7 +126,7 @@ const emitChatCompletions = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.chatCompletions.rendered': renderSSE(frames, back['ingress.chat.chatCompletions.wantsUsageChunk']), + 'response.chat.chatCompletions.rendered': move(renderSSE(frames, back['ingress.chat.chatCompletions.wantsUsageChunk'])), 'response.http.status': 200, }; }, From 049655ca4563e182c2703951f880c001f92cc62f Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:56:06 +0800 Subject: [PATCH 005/156] fix(chat): dial with the payload affinity materialized for the candidate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chain declared the services affinity needs and then read the client's own payload straight out of the record, so the per-candidate rewrite was lost — a turn carrying state for one upstream would have been sent to another unchanged. The ending now asks for what affinity materialized, and records the candidate that answered so a follow-up turn comes back to it. --- .../chat/chatcompletions-pipeline_test.ts | 25 +- .../chat/chat-completions/pipeline.ts | 12 +- .../src/data-plane/chat/gemini/pipeline.ts | 333 ++++++++++++++++++ .../src/data-plane/chat/messages/pipeline.ts | 309 ++++++++++++++++ 4 files changed, 674 insertions(+), 5 deletions(-) create mode 100644 packages/gateway/src/data-plane/chat/gemini/pipeline.ts create mode 100644 packages/gateway/src/data-plane/chat/messages/pipeline.ts diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index f9ef17b78c..2b4407fbf1 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -23,7 +23,7 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal let live: readonly ModelCandidate[] = []; const candidate = ( - callChatCompletions: () => Promise>, + callChatCompletions: (model: unknown, body: unknown) => Promise>, upstreamId = 'up_a', ): ModelCandidate => { const endpoints = { chatCompletions: {} }; @@ -63,6 +63,10 @@ const stream = (...events: readonly ChatCompletionsStreamEvent[]): ProviderStrea const payload = { model: 'chat-model', messages: [{ role: 'user', content: 'hi' }] } as unknown as ChatCompletionsPayload; +/** What affinity materialized for the candidate about to be dialled. It differs from the + * payload the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: ChatCompletionsPayload = payload; + const serve = async (wantsStream: boolean) => await run( chatCompletionsServePipeline(payload), move({ @@ -78,7 +82,7 @@ const serve = async (wantsStream: boolean) => await run( background: () => {}, rememberCandidates: () => {}, rememberChatSelection: () => {}, - chatPayloadFor: () => undefined, + chatPayloadFor: () => affinityPayload, selectAffinity: () => {}, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); @@ -97,6 +101,23 @@ beforeEach(() => { }); describe('the chat completions chain', () => { + // Affinity rewrites client-carried state for the upstream that will see it, so the body + // that goes out is the one it materialized rather than the one the client sent. + it('sends the payload affinity materialized for the candidate it dialled', async () => { + let sent: Record | undefined; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as ChatCompletionsPayload; + resolves([candidate(async (_model, body) => { + sent = body as Record; + return stream(chunk('hi')); + })]); + + await serve(false); + + expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + expect(sent).not.toHaveProperty('model'); + affinityPayload = payload; + }); + it('writes the frames out when the client asked to stream', async () => { resolves([candidate(async () => stream(chunk('he'), chunk('llo')))]); diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 4e5b06acdd..c57840400e 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -173,9 +173,12 @@ const callChatCompletionsUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - // The id the client addressed does not travel: the provider re-stamps whatever it - // resolved upstream, and an alias' own rules are applied to the body that is sent. - const payload = { ...facts['request.chat.chatCompletions'], model: candidate.model.id }; + // Affinity materializes the payload this candidate is owed: client-carried state is + // rewritten for the upstream that will see it, which is the whole reason a turn can be + // pinned at all. The id the client addressed does not travel — the provider re-stamps + // whatever it resolved upstream — and an alias' own rules apply to the body that is sent. + const asked = use.chatPayloadFor(facts['route.attempt']) as ChatCompletionsPayload; + const payload = { ...asked, model: candidate.model.id }; if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(payload, candidate.rules); const { model: _addressed, ...body } = payload; @@ -226,6 +229,9 @@ const callChatCompletionsUpstream = defineStage< }); } + // This candidate answered, so it is the one a follow-up turn carrying our own state + // must come back to. + use.selectAffinity(candidate); const metered = meterChatCompletions(result.events, identity); return move({ ...facts, diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts new file mode 100644 index 0000000000..5f0897eb2c --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -0,0 +1,333 @@ +// Gemini as a pipeline — the same chain as Chat Completions, for the one source-only family. +// +// emitGemini the edge: writes the answer in the shape the client asked for +// writeSettlement above the fork, so a run bills once however many wires it tried +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// callGeminiViaChatCompletions the ending: translates, dials, and meters what came back +// +// Gemini is source-only. Nothing translates *into* it, so it contributes one pipeline rather +// than two — a source role and no target role, and no other family's chain ever hands up +// `response.chat.gemini`. +// +// It is also the one family with no wire of its own: the provider surface has no `callGemini`, +// so every candidate is reached through a translation and the wire is whichever of the three +// translatable ones the picker finds on it. Only the first, Chat Completions, is built here. +// +// What is not built, stated rather than implied: +// - the Messages and the Responses wires. Each is its own ending stage — a different +// translate pair, a different provider method, and a different event dialect to meter on +// — and until they exist a candidate that `canServe` admitted on one of those two is +// dialled on Chat Completions anyway, which is the same interim state the Chat +// Completions chain carries for its own two translated wires. +// - `:countTokens`. It is a second entry against this protocol rather than another wire of +// this one: no stream, no billable turn, and a `{ totalTokens }` envelope of its own. +// - the Gemini interceptors and the affinity egress. The four `interceptors/` entries and +// the thought-signature rewrite on the way out are still only in the interceptor form. +// - a stream that ends without a terminal event. `collectGeminiProtocolEventsToResult` +// throws it, and here it leaves the run as a throw rather than as the 502 Google-RPC +// envelope `respond.ts` renders. +// +// One deliberate difference from `respond.ts`, shared with every other family on the +// pipeline: an upstream that refused is answered in its own words. A Chat Completions wire +// refuses in the OpenAI envelope, and that object is handed on as it came rather than being +// quoted back inside a Google-RPC `{ error: { code, message, status } }` — which also means +// the status the upstream sent is the status the client sees, with no coercion of the codes +// that envelope maps to `INTERNAL`. + +import { analyzeGeminiAffinity } from './affinity/ingress.ts'; +import type { UsageQuantities } from '../../../repo/types.ts'; +import type { BillableEntity } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { failover } from '../../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; +import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import { billableUsageFromChatCompletionsEvent } from '../chat-completions/usage.ts'; +import type { ChatFacts } from '../facts.ts'; +import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import { renderErrorEnvelope, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { + collectGeminiProtocolEventsToResult, + geminiProtocolFrameToSSEFrame, + type GeminiPayload, + type GeminiStreamEvent, +} from '@floway-dev/protocols/gemini'; +import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { translateGeminiViaChatCompletions } from '@floway-dev/translate'; + +/** `:generateContent` has no wire of its own, so the whole preference list is translated: + * Chat Completions first, then Messages, then Responses. */ +export const geminiTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); + +/** What this family adds to the chat space. */ +export interface GeminiFacts extends ChatFacts { + /** What the client is actually sent — an object when it asked for one, SSE frames when it + * asked to stream. The edge provides it, so a dump shows what the client received. */ + 'response.chat.gemini.rendered': Record | AsyncIterable; + /** What the upstream will have reported once the frames run out, and `null` when nothing + * streamed. Settling from this is the epilogue's job, after the drain. */ + 'response.chat.gemini.streamedUsage': Promise | null; +} + +type C = { [P in K]: GeminiFacts[P] }; + +/** + * The outermost edge. A Gemini answer is always a stream by the time it reaches here — the + * wire below speaks SSE whatever the client asked for, and the translation back into Gemini + * is itself a stream of frames — so what this decides is whether the client sees the frames + * or the one object they add up to. + * + * Collecting is therefore the edge's own work and not a second reading of the upstream: the + * same frames that would have gone out are folded here instead. + */ +const emitGemini = defineStage< + C<'ingress.chat.gemini.wantsStream'>, + C<'ingress.chat.gemini.wantsStream'>, + C<'ingress.chat.gemini.wantsStream' | 'response.chat.gemini' | 'response.http.headers'>, + C<'response.chat.gemini.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitGemini', + through: { + request: { + needs: ['ingress.chat.gemini.wantsStream'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.chat.gemini', 'response.http.headers'], + consumes: ['response.chat.gemini', 'response.http.headers'], + provides: ['response.chat.gemini.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.chat.gemini': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.gemini.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.http.status': answer.status, + }; + } + if (answer.kind === 'value') { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.gemini.rendered': move(answer.body as Record), + 'response.http.status': 200, + }; + } + + const frames = answer.frames as AsyncIterable>; + if (!back['ingress.chat.gemini.wantsStream']) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.gemini.rendered': move( + await collectGeminiProtocolEventsToResult(frames) as unknown as Record, + ), + 'response.http.status': 200, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.gemini.rendered': renderSSE(frames), + 'response.http.status': 200, + }; + }, +}); + +/** Gemini streams one JSON object per `data:` line and has no sentinel to write, which the + * protocol says by writing no frame at all for the one that ends the stream. */ +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + const written = geminiProtocolFrameToSSEFrame(frame); + if (written !== null) yield written; + } + })(), +}); + +/** + * The Chat Completions wire. It translates the turn, dials, and hands the answer up at this + * family's own response key — which is what makes it interchangeable with the two wires that + * are not built yet: all three would provide `response.chat.gemini`, and the stage above + * cannot tell which ran. + * + * The reading is taken on the dialect the upstream actually spoke. Usage is metered off the + * Chat Completions frames on their way into the translation rather than off the Gemini frames + * that come out of it, so a figure the upstream stated is billed as stated. + */ +const callGeminiViaChatCompletions = defineStage< + C<'request.chat.gemini' | 'route.attempt' | 'ingress.http.headers'>, + C<'response.chat.gemini' | 'response.chat.gemini.streamedUsage' + | 'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'callGeminiViaChatCompletions', + return: { + provides: [ + 'response.chat.gemini', + 'response.chat.gemini.streamedUsage', + 'response.usage.billable', + 'response.http.headers', + ], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // Attribution is set before the dial — before the translation too, which can reject the + // client's own input — so an attempt that never completes still names the candidate it was + // made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + + // The trip owns both directions: it builds the payload that goes out, stamping the id the + // candidate resolved to rather than the one the client addressed in its URL, and it hands + // back the closure that turns the upstream's frames into Gemini's. An alias' own rules are + // applied to the translated body, which is the only shape the wire will see. + const trip = await translateGeminiViaChatCompletions(facts['request.chat.gemini'], { model: candidate.model.id }); + if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(trip.target, candidate.rules); + const { model: _addressed, ...body } = trip.target; + + let result; + try { + result = await candidate.provider.instance.callChatCompletions( + providerModelOf(candidate), + body, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.chat.gemini': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.gemini.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.headers': [], + }); + } + + const identity = telemetryModelIdentity(candidate, result.modelKey); + // An upstream that was called and reported nothing, which is a different statement from + // reporting zero. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + + if (!result.ok) { + const text = await result.response.text(); + use.log.warn('upstream refused', { status: result.response.status }); + let parsed: unknown; + try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } + return move({ + ...facts, + 'response.chat.gemini': { + status: result.response.status, + message: text, + ...(parsed === undefined ? {} : { body: parsed }), + }, + 'response.chat.gemini.streamedUsage': null, + 'response.usage.billable': called, + 'response.http.headers': [...result.response.headers], + }); + } + + const metered = meterChatCompletions(result.events, identity); + return move({ + ...facts, + 'response.chat.gemini': { kind: 'stream' as const, frames: trip.events(metered.frames) }, + 'response.chat.gemini.streamedUsage': metered.billable, + 'response.usage.billable': called, + 'response.http.headers': [...(result.headers ?? new Headers())], + }); + }, +}); + +/** Reads the upstream's own usage off its own events as they pass, so the reading costs one + * pass and the client's stream is what drives it. Only a report carrying real counts + * replaces the running figure, so a trailing empty usage frame cannot wipe a good one. */ +const meterChatCompletions = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, +): { readonly frames: AsyncIterable>; readonly billable: Promise } => { + let settle!: (billable: readonly BillableEntity[]) => void; + const billable = new Promise(resolve => { settle = resolve; }); + const generator = (async function* () { + let reported: BillableUsage | undefined; + try { + for await (const frame of source) { + if (frame.type === 'event') { + const usage = billableUsageFromChatCompletionsEvent(frame.event); + if (usage !== null) reported = usage; + } + yield frame; + } + } finally { + // Reached however the frames ended — the terminal chunk, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + settle([{ identity, quantities: billed(reported) }]); + } + })(); + return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; +}; + +/** An upstream that reported nothing leaves no quantities at all, which is a different + * statement from reporting zero. */ +const billed = (usage: BillableUsage | undefined): UsageQuantities => + usage === undefined ? {} : tokenUsageFromBillableUsage(usage) as unknown as UsageQuantities; + +/** A candidate that cannot serve *this* request is not a candidate — and what the client's + * own turn carries decides the order the rest are tried in, which is why the narrowing is + * built from the request rather than being a constant. */ +const narrowing = (payload: GeminiPayload): ChatNarrowing> => ({ + source: 'gemini', + canServe: candidate => geminiTarget.canServe(candidate.model.endpoints), + affinity: async gateway => await analyzeGeminiAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the Gemini generateContent endpoint.`, + refuse: (status, message) => ({ 'response.chat.gemini': { status, message } }), + refuses: ['response.chat.gemini'], +}); + +export type GeminiServeEntry = C< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.gemini.wantsStream' + | 'request.chat.gemini' | 'serve.model' +>; + +export type GeminiServeExit = C< + 'response.chat.gemini.rendered' | 'response.chat.gemini.streamedUsage' + | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' +>; + +export const geminiServePipeline = (payload: GeminiPayload): Pipeline => + compose('geminiServe', [ + emitGemini, + writeSettlement( + handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini']), + handedUp => (handedUp as { 'response.chat.gemini.streamedUsage'?: unknown })['response.chat.gemini.streamedUsage'] !== null, + ), + resolveChatCandidates(narrowing(payload)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini']), + owns: [], + }), + callGeminiViaChatCompletions, + ]); diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts new file mode 100644 index 0000000000..b05e552514 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -0,0 +1,309 @@ +// Messages as a pipeline, on the chain Chat Completions established. +// +// emitMessages the edge: writes the answer in the shape the client asked for +// writeSettlement above the fork, so a run bills once however many wires it tried +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// callMessagesUpstream the ending: dials this candidate's wire +// +// Only the native wire is here. The translated ones — Messages via Responses and Messages +// via Chat Completions — hand up the same `response.chat.messages` key, which is what will +// make them interchangeable with this one, and each is a step of its own. So is +// `/v1/messages/count_tokens`: a second operation over this protocol rather than another +// wire under this pipeline. This family's own interceptors are not stages yet either, so +// the array between the fork and the ending is empty rather than short. + +import { analyzeMessagesAffinity } from './affinity/ingress.ts'; +import { createMessagesBillableUsageReader } from './usage.ts'; +import type { UsageQuantities } from '../../../repo/types.ts'; +import type { BillableEntity } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { failover } from '../../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; +import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import type { ChatFacts } from '../facts.ts'; +import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { renderErrorEnvelope, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { + collectMessagesProtocolEventsToResult, + messagesProtocolFrameToSSEFrame, + parseAnthropicBetaHeader, + type MessagesPayload, + type MessagesStreamEvent, +} from '@floway-dev/protocols/messages'; +import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; + +/** `/v1/messages` prefers its own wire, then the translated Responses path, then the + * translated Chat Completions path. */ +export const messagesTarget = chatTargetPicker(['messages', 'responses', 'chat-completions']); + +/** What this family adds to the chat space. */ +export interface MessagesFacts extends ChatFacts { + /** What the client is actually sent — an object when it asked for one, SSE frames when it + * asked to stream. The edge provides it, so a dump shows what the client received. */ + 'response.chat.messages.rendered': Record | AsyncIterable; + /** What the upstream will have reported once the frames run out, and `null` when nothing + * streamed. Settling from this is the epilogue's job, after the drain. */ + 'response.chat.messages.streamedUsage': Promise | null; +} + +type M = { [P in K]: MessagesFacts[P] }; + +/** + * The outermost edge. A Messages answer is always a stream by the time it reaches here — + * the upstream speaks SSE whatever the client asked for — so what this decides is whether + * the client sees the frames or the one message they add up to. + * + * Collecting is therefore the edge's own work and not a second reading of the upstream: the + * same frames that would have gone out are folded here instead, by the protocol's own + * reassembly, which is what makes a stream that stopped short of `message_stop` say so + * rather than answer with half a message. + */ +const emitMessages = defineStage< + M<'ingress.chat.messages.wantsStream'>, + M<'ingress.chat.messages.wantsStream'>, + M<'ingress.chat.messages.wantsStream' | 'response.chat.messages' | 'response.http.headers'>, + M<'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitMessages', + through: { + request: { needs: ['ingress.chat.messages.wantsStream'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.messages', 'response.http.headers'], + consumes: ['response.chat.messages', 'response.http.headers'], + provides: ['response.chat.messages.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.chat.messages': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.messages.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.http.status': answer.status, + }; + } + if (answer.kind === 'value') { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.messages.rendered': move(answer.body as Record), + 'response.http.status': 200, + }; + } + + const frames = answer.frames as AsyncIterable>; + if (!back['ingress.chat.messages.wantsStream']) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.messages.rendered': move( + await collectMessagesProtocolEventsToResult(frames) as unknown as Record, + ), + 'response.http.status': 200, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.messages.rendered': renderSSE(frames), + 'response.http.status': 200, + }; + }, +}); + +/** Anthropic names its own SSE events, and every frame that has one is the client's. Which + * frames have an SSE form at all is the protocol's to say, and it says so by writing no + * frame — a stream terminator is a Chat Completions idea and there is nothing to send for + * it here. */ +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + const written = messagesProtocolFrameToSSEFrame(frame); + if (written !== null) yield written; + } + })(), +}); + +/** + * The native wire. It dials Messages and provides the answer at this family's own response + * key — which is what will make it interchangeable with a translated chain: both hand up + * `response.chat.messages`, and the stage above cannot tell which ran. + */ +const callMessagesUpstream = defineStage< + M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers'>, + M<'response.chat.messages' | 'response.chat.messages.streamedUsage' + | 'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'callMessagesUpstream', + return: { + provides: [ + 'response.chat.messages', + 'response.chat.messages.streamedUsage', + 'response.usage.billable', + 'response.http.headers', + ], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + + // The id the client addressed does not travel: the provider re-stamps whatever it + // resolved upstream, and an alias' own rules are applied to the body that is sent. + const payload = { ...facts['request.chat.messages'], model: candidate.model.id }; + if (candidate.rules !== undefined) applyRulesToUpstreamMessages(payload, candidate.rules); + const { model: _addressed, ...body } = payload; + + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. Anthropic's beta flags are the exception — they have a + // typed path of their own so no header allowlist can admit them and no other source + // protocol can leak them in, which is why the field is read off and then dropped. + const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); + const anthropicBeta = parseAnthropicBetaHeader(headers.get('anthropic-beta')); + headers.delete('anthropic-beta'); + + let result; + try { + result = await candidate.provider.instance.callMessages( + providerModelOf(candidate), + body, + use.gateway.abortSignal, + { ...buildUpstreamCallOptions(candidate, use.gateway, headers), anthropicBeta }, + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.chat.messages': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.messages.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.headers': [], + }); + } + + const identity = telemetryModelIdentity(candidate, result.modelKey); + // An upstream that was called and reported nothing, which is a different statement from + // reporting zero. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + + if (!result.ok) { + const text = await result.response.text(); + use.log.warn('upstream refused', { status: result.response.status }); + let parsed: unknown; + try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } + return move({ + ...facts, + 'response.chat.messages': { + status: result.response.status, + message: text, + ...(parsed === undefined ? {} : { body: parsed }), + }, + 'response.chat.messages.streamedUsage': null, + 'response.usage.billable': called, + 'response.http.headers': [...result.response.headers], + }); + } + + const metered = meterMessages(result.events, identity); + return move({ + ...facts, + 'response.chat.messages': { kind: 'stream' as const, frames: metered.frames }, + 'response.chat.messages.streamedUsage': metered.billable, + 'response.usage.billable': called, + 'response.http.headers': [...(result.headers ?? new Headers())], + }); + }, +}); + +/** Reads the upstream's own usage off its own events as they pass, so the reading costs one + * pass and the client's stream is what drives it. Anthropic states input accounting on + * `message_start` and output accounting on `message_delta`, so the reader that merges the + * two is per-stream state and is made here rather than shared. */ +const meterMessages = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, +): { readonly frames: AsyncIterable>; readonly billable: Promise } => { + let settle!: (billable: readonly BillableEntity[]) => void; + const billable = new Promise(resolve => { settle = resolve; }); + const readBillableUsage = createMessagesBillableUsageReader(); + const generator = (async function* () { + let reported: BillableUsage | undefined; + try { + for await (const frame of source) { + if (frame.type === 'event') { + const usage = readBillableUsage(frame.event); + if (usage !== null) reported = usage; + } + yield frame; + } + } finally { + // Reached however the frames ended — the terminal event, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + settle([{ identity, quantities: billed(reported) }]); + } + })(); + return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; +}; + +/** An upstream that reported nothing leaves no quantities at all, which is a different + * statement from reporting zero. */ +const billed = (usage: BillableUsage | undefined): UsageQuantities => + usage === undefined ? {} : tokenUsageFromBillableUsage(usage) as unknown as UsageQuantities; + +/** A candidate that cannot serve *this* request is not a candidate — and what the client's + * own turn carries decides the order the rest are tried in, which is why the narrowing is + * built from the request rather than being a constant. */ +const narrowing = (payload: MessagesPayload): ChatNarrowing> => ({ + source: 'messages', + canServe: candidate => messagesTarget.canServe(candidate.model.endpoints), + affinity: async gateway => await analyzeMessagesAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the /messages endpoint.`, + refuse: (status, message) => ({ 'response.chat.messages': { status, message } }), + refuses: ['response.chat.messages'], +}); + +export type MessagesServeEntry = M< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.messages.wantsStream' + | 'request.chat.messages' | 'serve.model' +>; + +export type MessagesServeExit = M< + 'response.chat.messages.rendered' | 'response.chat.messages.streamedUsage' + | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' +>; + +export const messagesServePipeline = (payload: MessagesPayload): Pipeline => + compose('messagesServe', [ + emitMessages, + writeSettlement( + handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages']), + handedUp => (handedUp as { 'response.chat.messages.streamedUsage'?: unknown })['response.chat.messages.streamedUsage'] !== null, + ), + resolveChatCandidates(narrowing(payload)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages']), + owns: [], + }), + callMessagesUpstream, + ]); From 713f400a4f993abb718d9785a333d4d524a2fea0 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:00:51 +0800 Subject: [PATCH 006/156] feat(chat): open a chat run over the context a chat run needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A chat run's three extra services are all the live half of resolution: the candidate, and the payload affinity materializes for it. Neither can enter the record — move() would freeze the provider's own models cache, and materializing is per candidate — so the resolver keeps them and the stage that dials asks back by selector, which is the shape the shared services already use. The seam splits rather than being wrapped. Building the chat context over an already-built one would have minted a second dump accumulator, and the body can only be taken once: takeRequestBody empties what it is given, so the second call would hand the dump an empty buffer. What is shared is now the options and the services, and each family builds the one context its run has. --- .../gateway/src/data-plane/chat/prologue.ts | 58 +++++++++++++++++++ .../gateway/src/data-plane/pipeline/serve.ts | 27 ++++++--- 2 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 packages/gateway/src/data-plane/chat/prologue.ts diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts new file mode 100644 index 0000000000..c00ba96dc2 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -0,0 +1,58 @@ +// Opening a chat run. The seam every non-chat family uses builds the services a run is +// given; a chat run needs three more, and this is where they come from. +// +// All three are the live half of resolution. A `ModelCandidate` never enters the record — +// `move()` would freeze the provider's own models cache — and neither does the payload +// affinity materializes for one, because materializing rewrites client-carried state for the +// upstream that will see it and the result is per candidate. So the resolver keeps them and +// the stage that dials asks for them back by selector, which is the same shape the shared +// services already use for the candidate itself. + +import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from './shared/gateway-ctx.ts'; +import type { ChatServices } from './stages.ts'; +import type { ApiKey } from '../../repo/types.ts'; +import type { AttemptSelector } from '../pipeline/facts.ts'; +import { gatewayCtxOptions, prologueFor, type Ingress, type Prologue } from '../pipeline/serve.ts'; +import type { StatefulResponsesStore } from './responses/items/store.ts'; +import type { AuthedContext } from '../../middleware/auth.ts'; +import type { ModelCandidate } from '@floway-dev/provider'; + +export interface ChatPrologue extends Prologue { + readonly services: ChatServices; + readonly gateway: ChatGatewayCtx; +} + +export const openChatPrologue = ( + c: AuthedContext, + ingress: Ingress, + options: { + readonly wantsStream: boolean; + readonly model?: string; + /** Native Responses entries persist their items; every other source gets a scratchpad, + * so the server-tool shim's request-private state always has a home. */ + readonly storeFactory: (apiKey: ApiKey, requestStartedAt: number) => StatefulResponsesStore; + }, +): ChatPrologue => { + // One context, built once: the body is read once and `takeRequestBody` empties what it was + // given, so the chat context is what this run *has* rather than a second one over it. + const gateway = createChatGatewayCtxFromHono(c, gatewayCtxOptions(c, ingress, options), options.storeFactory); + const base = prologueFor(gateway, ingress); + + let materialize: ((candidate: ModelCandidate) => unknown) | undefined; + return { + ...base, + gateway, + services: { + ...base.services, + gateway, + rememberChatSelection: payloadFor => { materialize = payloadFor; }, + chatPayloadFor: (selector: AttemptSelector) => { + if (materialize === undefined) { + throw new Error('chatPayloadFor: nothing was resolved in this run; the selector did not come from it'); + } + return materialize(base.services.resolveAttempt(selector)); + }, + selectAffinity: candidate => { gateway.affinity.select(candidate); }, + }, + }; +}; diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 7834ae1080..22fcf4d3b9 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -15,7 +15,7 @@ import type { GatewayServices } from './services.ts'; import { settleBillable } from './settlement.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { consoleLogSink } from '../../runtime/log.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; +import { createGatewayCtxFromHono, finalizeGatewayResponse, type CreateGatewayCtxOptions, type GatewayCtx } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; import { writeSSEFrames } from '../shared/sse.ts'; import type { Pipeline } from '@floway-dev/pipeline'; @@ -66,14 +66,25 @@ export const openPrologue = ( c: Context, ingress: Ingress, options: { readonly wantsStream: boolean; readonly model?: string }, -): Prologue => { - const gateway = createGatewayCtxFromHono(c, { - wantsStream: options.wantsStream, - ...(options.model === undefined ? {} : { model: options.model }), - requestBody: takeRequestBody(ingress.body), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); +): Prologue => prologueFor(createGatewayCtxFromHono(c, gatewayCtxOptions(c, ingress, options)), ingress); +/** What a run's request context is built from. Exported because a family whose context is a + * richer one builds that instead, and both have to be built from the same read of the body: + * `takeRequestBody` empties what it is given, so calling this twice would hand the dump an + * empty buffer the second time. */ +export const gatewayCtxOptions = ( + c: Context, + ingress: Ingress, + options: { readonly wantsStream: boolean; readonly model?: string }, +): CreateGatewayCtxOptions => ({ + wantsStream: options.wantsStream, + ...(options.model === undefined ? {} : { model: options.model }), + requestBody: takeRequestBody(ingress.body), + backgroundScheduler: backgroundSchedulerFromContext(c), +}); + +/** The services every run is given, over whichever context it was opened with. */ +export const prologueFor = (gateway: GatewayCtx, ingress: Ingress): Prologue => { const live = new Map(); return { From 4c853d14f770d21ccaf2eeb34b4533f4ed7aa1a7 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:07:39 +0800 Subject: [PATCH 007/156] feat(gemini): run the source-only chain on the pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gemini contributes one pipeline rather than two, because nothing translates into it, and it has no wire of its own: the ending translates the turn out to Chat Completions, dials, and translates the answer back, metering on the dialect the upstream actually spoke. Only that first wire is built — the Messages and Responses ones, `:countTokens`, this family's interceptors and the affinity egress are named in the header as absent rather than left implied. Running the chain settled three things it had wrong. The stream handed to the client is moved into the record like every other fact; the turn that is translated is the one affinity materialized for the candidate, and the candidate that answered is marked for the next turn; and the metered reading is converted to billing metrics rather than cast to them, since a cast leaves a TokenUsage where UsageQuantities is declared and the usage row it writes then carries no metric at all. --- .../data-plane/chat/gemini-pipeline_test.ts | 296 ++++++++++++++++++ .../src/data-plane/chat/gemini/pipeline.ts | 35 ++- 2 files changed, 321 insertions(+), 10 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts new file mode 100644 index 0000000000..1cd6aba658 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -0,0 +1,296 @@ +// Gemini's pipeline, assembled and run. Gemini is the source-only family and the one with no +// wire of its own, so what a run of it has to show beyond the entry contract is the round +// trip: a turn translated out to Chat Completions, an answer translated back, and the reading +// taken on the dialect the upstream actually spoke rather than on the one the client did. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import type { GeminiPayload } from '@floway-dev/protocols/gemini'; +import { directFetcher, type ModelCandidate, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +/** The live candidates the resolver hands back. They never enter the record: a candidate + * carries the provider's instance, its fetcher and its models cache, and freezing those is + * what putting one in the record would do. What travels is the selector. */ +let live: readonly ModelCandidate[] = []; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] }); +}; + +const resolveAttempt = (selector: { readonly upstreamId: string }): ModelCandidate => { + const found = live.find(candidate => candidate.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; +}; + +type CallChatCompletions = ( + model: unknown, + body: unknown, + signal: AbortSignal | undefined, + opts: UpstreamCallOptions, +) => Promise>; + +const candidate = (upstream: string, callChatCompletions: CallChatCompletions): ModelCandidate => { + const endpoints = { chatCompletions: {} }; + return { + provider: { + upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callChatCompletions: callChatCompletions as never }), + }, + model: stubInternalModel({ id: 'gemini-2.5-pro', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'gemini-2.5-pro', endpoints }) } }, upstream), + fetcher: directFetcher, + }; +}; + +const chunk = (delta: Record, finishReason?: string): ProtocolFrame => + eventFrame({ + id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'gemini-2.5-pro', + choices: [{ index: 0, delta, ...(finishReason === undefined ? {} : { finish_reason: finishReason }) }], + } as ChatCompletionsStreamEvent); + +const usageChunk: ProtocolFrame = eventFrame({ + id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'gemini-2.5-pro', + choices: [], + usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 }, +} as ChatCompletionsStreamEvent); + +/** One whole turn on the wire below: two content deltas, the figures the upstream metered, + * and the sentinel that ends it. */ +const turn = (): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + yield chunk({ content: 'he' }); + yield chunk({ content: 'llo' }, 'stop'); + yield usageChunk; + yield doneFrame(); + })(), +}); + +const streamed = ( + events: AsyncIterable>, + headers?: Headers, +): ProviderStreamResult => ({ + ok: true, events, modelKey: 'gemini-2.5-pro-key', ...(headers === undefined ? {} : { headers }), +}); + +const payload = { contents: [{ role: 'user' as const, parts: [{ text: 'hello' }] }] }; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * turn the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: GeminiPayload = payload; + +/** Which candidates the run said a follow-up turn must come back to. */ +const selected: ModelCandidate[] = []; + +// Settlement is part of every serve pipeline, and it writes. A test that drives a whole +// pipeline therefore needs somewhere for the row to go. +const recorded: { usage: unknown[]; performance: unknown[] } = { usage: [], performance: [] }; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + recorded.usage = []; + recorded.performance = []; + selected.length = 0; + affinityPayload = payload; + initRepo({ + usage: { record: async (row: unknown) => { recorded.usage.push(row); } }, + performance: { + recordNeutral: async (dims: unknown) => { recorded.performance.push(dims); }, + recordZeroOutputError: async (dims: unknown) => { recorded.performance.push(dims); }, + }, + } as never); +}); + +const serve = async (facts: Record) => await run( + geminiServePipeline(payload), + move(facts) as never, + { + gateway: mockChatGatewayCtx({ wantsStream: facts['ingress.chat.gemini.wantsStream'] === true }), + background: () => {}, + rememberCandidates: () => {}, + rememberChatSelection: () => {}, + chatPayloadFor: () => affinityPayload, + selectAffinity: (candidate: ModelCandidate) => { selected.push(candidate); }, + resolveAttempt, + } as never, +); + +const entryFacts = (overrides: Record = {}) => ({ + 'ingress.chat.gemini.wantsStream': true, + 'ingress.chat.sourceProtocol': 'gemini', + 'ingress.http.headers': [], + 'request.chat.gemini': payload, + 'serve.model': 'gemini-2.5-pro', + ...overrides, +}); + +const collect = async (rendered: unknown): Promise => { + const frames: SseFrame[] = []; + for await (const frame of rendered as AsyncIterable) frames.push(frame); + return frames; +}; + +describe('the gemini pipeline', () => { + // The whole entry contract, and it does not mention `request.chat.gemini` — the stage that + // reads the turn is the ending, a stage whose only trait is `return` declares no request + // side at all, and so assembly cannot see what it reads. A caller who omits the payload + // gets a runtime failure at the deepest stage rather than the assembly error the entry + // contract exists to give it. + it('assembles, and asks its caller only for what the descending stages need', () => { + expect([...geminiServePipeline(payload).entryNeeds].sort()).toEqual([ + 'ingress.chat.gemini.wantsStream', + 'serve.model', + ]); + }); + + it('translates the turn out, dials the Chat Completions wire, and streams Gemini frames back', async () => { + let sent: Record | undefined; + resolves([candidate('up_a', async (_model, body) => { + sent = body as Record; + return streamed(turn()); + })]); + + const { facts, drain } = await serve(entryFacts()); + + // The answer comes back before the drain runs, which is what lets a streaming family + // hand its stream on: the frames are still there to read. + expect((await collect(facts['response.chat.gemini.rendered'])).map(frame => JSON.parse(frame.data) as unknown)).toEqual([ + { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'he' }] } }] }, + { + candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'llo' }] }, finishReason: 'STOP' }], + usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 7, totalTokenCount: 12 }, + }, + ]); + // What went out is the translated turn, addressed to the candidate's own id — which the + // ending then strips, because the provider re-stamps what it resolved upstream. + expect(sent).toMatchObject({ stream: true, messages: [{ role: 'user', content: 'hello' }] }); + expect(sent).not.toHaveProperty('model'); + await drain(); + }); + + // Gemini carries its state in `thoughtSignature` parts, which only mean anything to the + // upstream that issued them — so the turn that is translated is the one affinity rewrote + // for this candidate, and the candidate that answered is marked for the next turn. + it('translates what affinity materialized, and marks the candidate that answered', async () => { + let sent: Record | undefined; + affinityPayload = { contents: [{ role: 'user', parts: [{ text: 'rewritten' }] }] }; + const dialled = candidate('up_a', async (_model, body) => { + sent = body as Record; + return streamed(turn()); + }); + resolves([dialled]); + + const { facts, drain } = await serve(entryFacts()); + await collect(facts['response.chat.gemini.rendered']); + + expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + expect(selected).toEqual([dialled]); + await drain(); + }); + + it('folds the frames into one Gemini result for a client that did not ask to stream', async () => { + resolves([candidate('up_a', async () => streamed(turn()))]); + + const { facts, drain } = await serve(entryFacts({ 'ingress.chat.gemini.wantsStream': false })); + + expect(facts['response.chat.gemini.rendered']).toEqual({ + candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'hello' }] }, finishReason: 'STOP' }], + usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 7, totalTokenCount: 12 }, + }); + expect(facts['response.http.status']).toBe(200); + await drain(); + }); + + it('fails a refusal over to the next candidate, and answers in the last upstream-s own words', async () => { + const tried: string[] = []; + resolves([ + candidate('up_a', async () => { + tried.push('up_a'); + return { ok: false, response: Response.json({ error: { message: 'slow down' } }, { status: 429 }), modelKey: 'k' }; + }), + candidate('up_b', async () => { + tried.push('up_b'); + return { ok: false, response: Response.json({ error: { message: 'no' } }, { status: 400 }), modelKey: 'k' }; + }), + ]); + + const { facts, drain } = await serve(entryFacts()); + + expect(tried).toEqual(['up_a', 'up_b']); + // Every candidate failed, so the last failure is the base: the client sees the status an + // upstream actually returned and the words that upstream used, rather than a synthesized + // envelope quoting its serialized body back as a message. + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.gemini.rendered']).toEqual({ error: { message: 'no' } }); + // An upstream that was called and reported nothing, which is a different statement from + // reporting zero. + expect(facts['response.usage.billable']).toEqual([ + { identity: { model: 'gemini-2.5-pro', upstream: 'up_b', modelKey: 'k', pricing: null }, quantities: {} }, + ]); + await drain(); + }); + + it('turns a dial that threw into a failure value rather than a thrown run', async () => { + resolves([candidate('up_a', () => Promise.reject(new Error('socket hang up')))]); + + const { facts, drain } = await serve(entryFacts()); + + expect(facts['response.http.status']).toBe(502); + expect(facts['response.chat.gemini.rendered']).toEqual({ error: { message: 'socket hang up', type: 'api_error' } }); + // A dial that never completed reached no upstream, so nothing was billed and there are no + // headers to carry. + expect(facts['response.usage.billable']).toEqual([]); + expect(facts['response.http.headers']).toEqual([]); + await drain(); + }); + + it('forwards the upstream headers a client may see and drops the ones it may not', async () => { + resolves([candidate('up_a', async () => streamed(turn(), new Headers({ + 'content-type': 'text/event-stream', + 'transfer-encoding': 'chunked', + 'x-request-id': 'req_1', + })))]); + + const { facts, drain } = await serve(entryFacts()); + + expect(facts['response.http.headers']).toEqual([['x-request-id', 'req_1']]); + await collect(facts['response.chat.gemini.rendered']); + await drain(); + }); + + // The wire below always streams, whatever the client asked for, so this family's numbers + // always arrive with the last chunk — which is after the run has answered. Settling in the + // stage as well would write the row twice, so the pipeline hands the numbers up as a promise + // and the epilogue is what writes them. + it('defers settlement to the promise it hands up, metered on the dialect the upstream spoke', async () => { + resolves([candidate('up_a', async () => streamed(turn()))]); + + const { facts, drain } = await serve(entryFacts()); + await collect(facts['response.chat.gemini.rendered']); + await drain(); + + expect(recorded.usage).toHaveLength(0); + const usage = facts['response.chat.gemini.streamedUsage']; + expect(usage).not.toBeNull(); + expect(await usage!).toEqual([ + { + identity: { model: 'gemini-2.5-pro', upstream: 'up_a', modelKey: 'gemini-2.5-pro-key', pricing: null }, + quantities: { input_tokens: '5', output_tokens: '7' }, + }, + ]); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 5f0897eb2c..d16413b5f8 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -37,6 +37,7 @@ import { analyzeGeminiAffinity } from './affinity/ingress.ts'; import type { UsageQuantities } from '../../../repo/types.ts'; +import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; @@ -146,7 +147,7 @@ const emitGemini = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': renderSSE(frames), + 'response.chat.gemini.rendered': move(renderSSE(frames)), 'response.http.status': 200, }; }, @@ -174,7 +175,7 @@ const renderSSE = (frames: AsyncIterable>): Asy * that come out of it, so a figure the upstream stated is billed as stated. */ const callGeminiViaChatCompletions = defineStage< - C<'request.chat.gemini' | 'route.attempt' | 'ingress.http.headers'>, + C<'route.attempt' | 'ingress.http.headers'>, C<'response.chat.gemini' | 'response.chat.gemini.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, ChatServices @@ -195,11 +196,15 @@ const callGeminiViaChatCompletions = defineStage< // made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - // The trip owns both directions: it builds the payload that goes out, stamping the id the - // candidate resolved to rather than the one the client addressed in its URL, and it hands - // back the closure that turns the upstream's frames into Gemini's. An alias' own rules are - // applied to the translated body, which is the only shape the wire will see. - const trip = await translateGeminiViaChatCompletions(facts['request.chat.gemini'], { model: candidate.model.id }); + // The trip owns both directions: it builds the payload that goes out and hands back the + // closure that turns the upstream's frames into Gemini's. What it translates is what + // affinity materialized for this candidate — client-carried state rewritten for the + // upstream that will see it, which is the whole reason a turn can be pinned at all. The + // id the client addressed does not travel: Gemini carries it in the URL, the trip stamps + // the candidate's own, and the wire is handed the body without it. An alias' own rules + // apply to the translated body, which is the only shape the wire will see. + const asked = use.chatPayloadFor(facts['route.attempt']) as GeminiPayload; + const trip = await translateGeminiViaChatCompletions(asked, { model: candidate.model.id }); if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(trip.target, candidate.rules); const { model: _addressed, ...body } = trip.target; @@ -250,6 +255,9 @@ const callGeminiViaChatCompletions = defineStage< }); } + // This candidate answered, so it is the one a follow-up turn carrying our own state must + // come back to. + use.selectAffinity(candidate); const metered = meterChatCompletions(result.events, identity); return move({ ...facts, @@ -291,9 +299,16 @@ const meterChatCompletions = ( }; /** An upstream that reported nothing leaves no quantities at all, which is a different - * statement from reporting zero. */ -const billed = (usage: BillableUsage | undefined): UsageQuantities => - usage === undefined ? {} : tokenUsageFromBillableUsage(usage) as unknown as UsageQuantities; + * statement from reporting zero. + * + * A billed entity is keyed by billing metric, which is not the shape a protocol reports in, + * so the reading is converted rather than cast. The service tier survives as far as + * `TokenUsage.tier` and no further: a billed entity is an identity and a bag of quantities, + * and the pricing selector the tier feeds has no seat there. */ +const billed = (usage: BillableUsage | undefined): UsageQuantities => { + const tokens = tokenUsageFromBillableUsage(usage); + return tokens === null ? {} : tokenUsageQuantities(tokens); +}; /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is From 1256077506ec8bb17591799700998d6c02080774 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:07:58 +0800 Subject: [PATCH 008/156] feat(chat): make a protocol handoff a stage that consumes and provides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A translation is two declarations: it consumes the source protocol's request key and provides the target's, and coming back it consumes the target's response key and provides the source's. Saying it that way retires the runtime test. The source key is gone below the handoff, so a stage needing it cannot be placed there and compose says so at assembly — which is ctx.targetApi === deleted and made structural. And nothing below knows a translation happened: the target chain sees its own protocol's keys and only those, so one chain serves a native request and a translated one. A refusal is handed to the pair as the upstream's own bytes, with the headers that actually came back rather than a synthesized set — that is what lets a context-window error become the shape Claude Code reads for auto-compaction. An answer that arrived as a value has no pair to map it, so the handoff fails rather than handing the client another protocol's object under its own key. --- .../gateway/src/data-plane/chat/handoff.ts | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 packages/gateway/src/data-plane/chat/handoff.ts diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts new file mode 100644 index 0000000000..a6a33a4a1b --- /dev/null +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -0,0 +1,112 @@ +// A protocol handoff, as a stage. +// +// This is the whole of what a translation is in the fact space: it **consumes** the source +// protocol's request key and **provides** the target's, and coming back it consumes the +// target's response key and provides the source's. Two consequences follow from saying it +// that way rather than carrying a `targetApi` field. +// +// A chain cannot re-enter its own protocol. The source key is gone below this stage, so a +// stage that needs it cannot be placed there and `compose` says so at assembly — which is +// `ctx.targetApi === ` deleted and made structural rather than tested at runtime. +// +// And nothing below knows a translation happened. The target chain sees its own protocol's +// keys and only those, so the same chain serves a native request and a translated one. + +import type { ChatAnswer, ChatFacts } from './facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { TranslateTripResult } from '@floway-dev/translate'; + +type RequestKey = Extract; +type ResponseKey = Extract; + +export interface Handoff { + /** The two request keys, and the two response keys they answer at. Naming all four is what + * lets the declaration be written; a pair is not derivable from one half. */ + readonly from: { readonly request: Source; readonly response: ResponseKey }; + readonly to: { readonly request: Target; readonly response: ResponseKey }; + /** The pair function, already closed over whatever context it reads. It returns the target + * payload together with the closure that maps the target's frames back — trip-scoped + * state (synthetic ids, custom-tool name sets) lives in that closure and the source chain + * never sees it. */ + readonly trip: (source: ChatFacts[Source]) => Promise>; +} + +/** + * Translates on the way down and back on the way up. + * + * An upstream refusal gets the pair's own rewrite where it has one: the canonical case is a + * context-window error becoming the Anthropic `prompt is too long:` shape that Claude Code + * reads for auto-compaction. A pair that declares no rewrite hands the refusal up unchanged. + */ +export const handOff = ( + handoff: Handoff, +) => defineStage< + { [K in Source]: ChatFacts[Source] }, + { [K in Target]: ChatFacts[Target] }, + { [K in ResponseKey]?: ChatAnswer } & Record, + { [K in ResponseKey]?: ChatAnswer } & Record +>({ + name: `handOff:${handoff.from.request}→${handoff.to.request}`, + through: { + request: { + needs: [handoff.from.request], + consumes: [handoff.from.request], + provides: [handoff.to.request], + }, + response: { + // The upstream's own headers are read, not rewritten: a pair that rewrites a refusal + // is handed what actually came back rather than a synthesized set. + needs: [handoff.to.response, 'response.http.headers'], + consumes: [handoff.to.response], + provides: [handoff.from.response], + }, + }, + execute: async (facts, next) => { + const record = facts as Record; + const { [handoff.from.request]: source, ...down } = record; + const trip = await handoff.trip(source as ChatFacts[Source]); + + const back = await next({ ...down, [handoff.to.request]: move(trip.target) } as never) as Record; + const { [handoff.to.response]: answered, ...up } = back; + const answer = answered as ChatAnswer; + + if (isFailure(answer)) { + // The pair works on the upstream's own bytes, which is what `message` holds: a refusal + // is read as text and parsed only for whoever wants the object. + const rewritten = trip.apiError?.({ + status: answer.status, + headers: new Headers(((up['response.http.headers'] ?? []) as readonly (readonly [string, string])[]).map(([name, value]): [string, string] => [name, value])), + body: new TextEncoder().encode(answer.message), + }); + if (rewritten === undefined) return { ...up, [handoff.from.response]: answer } as never; + const text = new TextDecoder().decode(rewritten.body); + let parsed: unknown; + try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } + return { + ...up, + [handoff.from.response]: move({ + status: rewritten.status, + message: text, + ...(parsed === undefined ? {} : { body: parsed }), + }), + } as never; + } + if (answer.kind === 'value') { + // A pair maps frames and nothing else, so an answer that arrived as a value has no + // mapping to cross the protocol boundary with. Handing it on would give the client + // another protocol's object under its own key, so this fails rather than mistranslates. + throw new Error( + `${handoff.to.request} answered with a value, and no pair maps one into ${handoff.from.request}`, + ); + } + return { + ...up, + [handoff.from.response]: move({ + kind: 'stream', + frames: trip.events(answer.frames as AsyncIterable>), + }), + } as never; + }, +}); From 3455e8284d9361ffe4b2033683952daa9ce96ca2 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:10:25 +0800 Subject: [PATCH 009/156] fix(chat): bill the metrics an upstream's usage adds up to, not its token categories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What an upstream reports is per token category and what is billed is per metric name; the two are different shapes and a cast between them typechecks. The usage row was therefore written with no metrics at all — the recorder looked for input_tokens and found input. Converting through the same function every other caller uses fixes it, and the chain now asserts the quantities rather than only that a row was written. --- .../chat/chatcompletions-pipeline_test.ts | 15 +++++++++++++++ .../data-plane/chat/chat-completions/pipeline.ts | 10 +++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index 2b4407fbf1..ff2ffdbfcd 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -171,6 +171,21 @@ describe('the chat completions chain', () => { expect(facts['response.chat.chatCompletions.rendered']).toEqual({ error: { message: 'slow down' } }); }); + // What the upstream reported is per token category and what is billed is per metric name; + // the two are not interchangeable, and a cast between them writes a row with no metrics. + it('bills the metrics the upstream-s own usage adds up to', async () => { + resolves([candidate(async () => stream(chunk('hi'), { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [], usage: { prompt_tokens: 11, completion_tokens: 4, total_tokens: 15 }, + } as unknown as ChatCompletionsStreamEvent))]); + + const { facts } = await serve(true); + for await (const _frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { /* drain */ } + const billable = await facts['response.chat.chatCompletions.streamedUsage']!; + + expect(billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); + }); + // A refused connection is an outcome the fork has to be able to see, not a fault that ends // the run — so the second candidate is tried and its answer is the one served. it('fails a dial that never connected over to the next candidate', async () => { diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index c57840400e..9e53f6b976 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -20,6 +20,7 @@ import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; @@ -273,9 +274,12 @@ const meterChatCompletions = ( }; /** An upstream that reported nothing leaves no quantities at all, which is a different - * statement from reporting zero. */ -const billed = (usage: BillableUsage | undefined): UsageQuantities => - usage === undefined ? {} : tokenUsageFromBillableUsage(usage) as unknown as UsageQuantities; + * statement from reporting zero. The two shapes are not interchangeable: what the upstream + * reported is per token category, and what is billed is per metric name. */ +const billed = (usage: BillableUsage | undefined): UsageQuantities => { + const tokens = tokenUsageFromBillableUsage(usage); + return tokens === null ? {} : tokenUsageQuantities(tokens); +}; /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is From 97c4a5246564d858ad443d7f0f4878babd93a9a0 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:13:26 +0800 Subject: [PATCH 010/156] fix(gemini): answer a refusal in the shape a Gemini client reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chain rendered failures through the shared envelope, which is the OpenAI families' shape: a Gemini SDK reads error.status — the Google-RPC name — and would have found it undefined, and error.code carries the HTTP status the generic shape has no field for. The mapping the replaced surface used already existed next door; it is now a renderer the edge calls, so the two entry points that build this envelope agree on one definition of it. --- .../__tests__/data-plane/chat/gemini-pipeline_test.ts | 4 ++-- packages/gateway/src/data-plane/chat/gemini/errors.ts | 7 +++++++ packages/gateway/src/data-plane/chat/gemini/pipeline.ts | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index 1cd6aba658..a38b9eba7f 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -235,7 +235,7 @@ describe('the gemini pipeline', () => { // upstream actually returned and the words that upstream used, rather than a synthesized // envelope quoting its serialized body back as a message. expect(facts['response.http.status']).toBe(400); - expect(facts['response.chat.gemini.rendered']).toEqual({ error: { message: 'no' } }); + expect(facts['response.chat.gemini.rendered']).toEqual({ error: { code: 400, message: '{"error":{"message":"no"}}', status: 'INVALID_ARGUMENT' } }); // An upstream that was called and reported nothing, which is a different statement from // reporting zero. expect(facts['response.usage.billable']).toEqual([ @@ -250,7 +250,7 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts()); expect(facts['response.http.status']).toBe(502); - expect(facts['response.chat.gemini.rendered']).toEqual({ error: { message: 'socket hang up', type: 'api_error' } }); + expect(facts['response.chat.gemini.rendered']).toEqual({ error: { code: 502, message: 'socket hang up', status: 'UNAVAILABLE' } }); // A dial that never completed reached no upstream, so nothing was billed and there are no // headers to carry. expect(facts['response.usage.billable']).toEqual([]); diff --git a/packages/gateway/src/data-plane/chat/gemini/errors.ts b/packages/gateway/src/data-plane/chat/gemini/errors.ts index 8b94d63a5b..b9def82698 100644 --- a/packages/gateway/src/data-plane/chat/gemini/errors.ts +++ b/packages/gateway/src/data-plane/chat/gemini/errors.ts @@ -7,6 +7,13 @@ import type { TranslatorInputError } from '@floway-dev/translate'; // Google RPC Status envelope, used by Gemini's `error` channel everywhere // (HTTP body, SSE-tunnelled error event). +/** What a Gemini client is sent when a turn produced no content. `error.status` is the + * Google-RPC name and clients read it, so this shape is the protocol's rather than the + * gateway's — an OpenAI-shaped envelope would leave that field undefined. */ +export const renderGeminiError = (status: number, message: string): Record => ({ + error: { code: status, message, status: geminiStatusForHttpStatus(status) }, +}); + export const geminiStatusForHttpStatus = (status: number): string => { switch (status) { case 400: diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index d16413b5f8..877b4538de 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -51,9 +51,10 @@ import type { ChatFacts } from '../facts.ts'; import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { renderGeminiError } from './errors.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { renderErrorEnvelope, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import type { BillableUsage, ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { collectGeminiProtocolEventsToResult, geminiProtocolFrameToSSEFrame, @@ -120,7 +121,7 @@ const emitGemini = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.chat.gemini.rendered': move(renderGeminiError(answer.status, answer.message)), 'response.http.status': answer.status, }; } From 703a21b68d03ddf13da882badfaddf6bf5a3581d Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:14:20 +0800 Subject: [PATCH 011/156] feat(chat): run the Messages chain on the pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /v1/messages served through the chain Chat Completions established: an edge that writes Anthropic's own named SSE events when the client asked to stream and reassembles them into one message when it did not, settlement above the fork, affinity-ordered candidates, and a native ending that dials callMessages. Three things the replaced serve/attempt/respond surface said are said here rather than left implied. The turn that goes out is the one affinity materialized for the candidate, and the candidate that answered is marked for the next turn. The beta flags keep their typed transport path — read off the inbound headers and dropped from what the provider is handed, so no header allowlist can admit them and no other source protocol can leak them in. And a dial that never connected is a failure value the fork moves past rather than a throw that ends the run. Only the native wire is built. The translated ones, count_tokens and this family's own interceptors are named in the header as absent rather than left implied. What the header cannot say is that a pre-upstream refusal now renders in the shared error envelope rather than Anthropic's, since renderErrorEnvelope is what the edge writes; an upstream that refused in its own words is still handed on in them. --- .../data-plane/chat/messages-pipeline_test.ts | 275 ++++++++++++++++++ .../src/data-plane/chat/messages/pipeline.ts | 29 +- 2 files changed, 297 insertions(+), 7 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts new file mode 100644 index 0000000000..a3975688c9 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts @@ -0,0 +1,275 @@ +// The Messages chain, run. Assembly is checked where every family's is; what is written +// down here is what only running it can say — that the edge writes Anthropic's own named +// SSE events when the client asked to stream and reassembles them into one message when it +// did not, that a refusal keeps the upstream's own status and words, that a dial nobody +// answered is a value the fork can move past, that the beta flags travel on their typed +// path rather than as a header, and that what the two usage-bearing events add up to is +// billed in the metrics settlement reads. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { messagesServePipeline } from '../../../src/data-plane/chat/messages/pipeline.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; +import { directFetcher, type MessagesUpstreamCallOptions, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +type CallMessages = ( + model: unknown, + body: unknown, + signal: AbortSignal | undefined, + opts: MessagesUpstreamCallOptions, +) => Promise>; + +const candidate = (callMessages: CallMessages, upstreamId = 'up_a'): ModelCandidate => { + const endpoints = { messages: {} }; + return { + provider: { + upstreamId, kind: 'claude-code', name: upstreamId, + // Wide enough that a header reaching the provider proves the stage let it through, + // rather than proving only that the allowlist did. + inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callMessages: callMessages as never }), + }, + model: stubInternalModel( + { id: 'claude-model', endpoints, providerModels: { [upstreamId]: stubProviderModel({ id: 'claude-model', endpoints }) } }, + upstreamId, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +// One well-formed turn: Anthropic states input accounting on `message_start` and output +// accounting on `message_delta`, so a stream that says what it cost says it in two places. +const messageStart = { + type: 'message_start', + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'claude-model', + stop_reason: null, stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 2 }, + }, +}; +const blockStart = { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; +const textDelta = (text: string) => ({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }); +const blockStop = { type: 'content_block_stop', index: 0 }; +const messageDelta = { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 7 } }; +const messageStop = { type: 'message_stop' }; + +const turn = (...text: readonly string[]): readonly unknown[] => + [messageStart, blockStart, ...text.map(textDelta), blockStop, messageDelta, messageStop]; + +const stream = (events: readonly unknown[]): ProviderStreamResult => ({ + ok: true, + modelKey: 'claude-model-key', + headers: new Headers({ 'x-request-id': 'req-1', 'content-length': '99' }), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event: event as MessagesStreamEvent }; + yield { type: 'done' as const }; + })(), +}); + +const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as MessagesPayload; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * payload the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: MessagesPayload = payload; + +const serve = async (wantsStream: boolean, headers: readonly (readonly [string, string])[] = []) => await run( + messagesServePipeline(payload), + move({ + 'ingress.http.headers': headers, + 'ingress.chat.sourceProtocol': 'messages', + 'ingress.chat.messages.wantsStream': wantsStream, + 'request.chat.messages': payload, + 'serve.model': 'claude-model', + }) as never, + { + gateway: mockChatGatewayCtx({ wantsStream }), + background: () => {}, + rememberCandidates: () => {}, + rememberChatSelection: () => {}, + chatPayloadFor: () => affinityPayload, + selectAffinity: () => {}, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, +); + +const collect = async (rendered: unknown): Promise => { + const frames: SseFrame[] = []; + for await (const frame of rendered as AsyncIterable) frames.push(frame); + return frames; +}; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + affinityPayload = payload; + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the messages chain', () => { + // Affinity rewrites client-carried state — a thinking signature only its issuer can read — + // for the upstream that will see it, so the body that goes out is the one it materialized + // rather than the one the client sent. + it('sends the payload affinity materialized for the candidate it dialled', async () => { + let sent: Record | undefined; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as MessagesPayload; + resolves([candidate(async (_model, body) => { + sent = body as Record; + return stream(turn('hi')); + })]); + + await serve(false); + + expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + expect(sent).not.toHaveProperty('model'); + }); + + // Anthropic's clients dispatch on the SSE event name, not on the payload alone, so the + // name each frame carries is part of the answer. + it('writes the frames out under their own event names when the client asked to stream', async () => { + resolves([candidate(async () => stream(turn('he', 'llo')))]); + + const { facts } = await serve(true); + const frames = await collect(facts['response.chat.messages.rendered']); + + expect(facts['response.http.status']).toBe(200); + expect(frames.map(frame => frame.event)).toEqual([ + 'message_start', 'content_block_start', 'content_block_delta', 'content_block_delta', + 'content_block_stop', 'message_delta', 'message_stop', + ]); + expect(frames.map(frame => frame.data)).toEqual(turn('he', 'llo').map(event => JSON.stringify(event))); + }); + + // The upstream speaks SSE whatever the client asked for, so a client that did not ask to + // stream is answered from the same frames — folded here rather than read a second time. + it('reassembles the frames into one message when the client did not', async () => { + resolves([candidate(async () => stream(turn('he', 'llo')))]); + + const { facts } = await serve(false); + const rendered = facts['response.chat.messages.rendered'] as { + readonly type: string; + readonly content: readonly { readonly text: string }[]; + readonly stop_reason: string; + readonly usage: { readonly input_tokens: number; readonly output_tokens: number }; + }; + + expect(facts['response.http.status']).toBe(200); + expect(rendered.type).toBe('message'); + expect(rendered.content.map(block => block.text)).toEqual(['hello']); + expect(rendered.stop_reason).toBe('end_turn'); + expect(rendered.usage).toMatchObject({ input_tokens: 5, output_tokens: 7 }); + }); + + // Content-length would misdescribe a body this gateway serialized itself; a vendor trace + // is what a client and an operator both need to correlate a turn. + it('forwards the upstream headers a client may see and drops the ones it may not', async () => { + resolves([candidate(async () => stream(turn('hi')))]); + + const { facts } = await serve(true); + + expect(Object.fromEntries(facts['response.http.headers'])).toMatchObject({ 'x-request-id': 'req-1' }); + expect(Object.fromEntries(facts['response.http.headers'])).not.toHaveProperty('content-length'); + }); + + it('answers an upstream refusal with its own status and words', async () => { + resolves([candidate(async () => ({ + ok: false, + modelKey: 'claude-model-key', + response: new Response(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + }))]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(429); + expect(facts['response.chat.messages.rendered']).toEqual({ + type: 'error', + error: { type: 'rate_limit_error', message: 'slow down' }, + }); + }); + + // A refused connection is an outcome the fork has to be able to see, not a fault that ends + // the run — so the second candidate is tried and its answer is the one served. + it('fails a dial that never connected over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); }, 'up_dead'), + candidate(async () => { tried.push('alive'); return stream(turn('hi')); }, 'up_alive'), + ]); + + const { facts } = await serve(false); + + expect(tried).toEqual(['dead', 'alive']); + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.messages.rendered']).toMatchObject({ type: 'message' }); + }); + + // Anthropic beta flags have a typed path of their own precisely so no provider's header + // allowlist can admit them and no other source protocol can leak them in — so the field is + // read off the client's request and does not travel beside itself. + it('hands the beta flags over on their own path and not as a header', async () => { + let seen: MessagesUpstreamCallOptions | undefined; + resolves([candidate(async (_model, _body, _signal, opts) => { + seen = opts; + return stream(turn('hi')); + })]); + + await serve(false, [ + ['anthropic-beta', 'oauth-2025-04-20,fine-grained-tool-streaming-2025-05-14'], + ['x-trace', 'abc'], + ]); + + expect(seen?.anthropicBeta).toEqual(['oauth-2025-04-20', 'fine-grained-tool-streaming-2025-05-14']); + expect(seen?.headers.get('anthropic-beta')).toBeNull(); + expect(seen?.headers.get('x-trace')).toBe('abc'); + }); + + // The reading is per billing metric, which is not the shape Anthropic reports in: input + // accounting arrives on `message_start` and output accounting on `message_delta`, and what + // settlement writes is what the two add up to after the client's own stream drove them out. + it('bills what the upstream metered across both of its usage events', async () => { + resolves([candidate(async () => stream(turn('hi')))]); + + const { facts } = await serve(true); + + // Before the drain: called, and reported nothing — the numbers arrive with the last + // frame, which is after this run has answered. + expect(facts['response.usage.billable']).toEqual([ + { identity: { model: 'claude-model', upstream: 'up_a', modelKey: 'claude-model-key', pricing: null }, quantities: {} }, + ]); + + await collect(facts['response.chat.messages.rendered']); + + expect(await facts['response.chat.messages.streamedUsage']).toEqual([ + { + identity: { model: 'claude-model', upstream: 'up_a', modelKey: 'claude-model-key', pricing: null }, + quantities: { input_tokens: '5', input_cache_read_tokens: '2', output_tokens: '7' }, + }, + ]); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index b05e552514..a5a2100c40 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -16,6 +16,7 @@ import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { createMessagesBillableUsageReader } from './usage.ts'; import type { UsageQuantities } from '../../../repo/types.ts'; +import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; @@ -120,7 +121,7 @@ const emitMessages = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': renderSSE(frames), + 'response.chat.messages.rendered': move(renderSSE(frames)), 'response.http.status': 200, }; }, @@ -165,9 +166,13 @@ const callMessagesUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - // The id the client addressed does not travel: the provider re-stamps whatever it - // resolved upstream, and an alias' own rules are applied to the body that is sent. - const payload = { ...facts['request.chat.messages'], model: candidate.model.id }; + // Affinity materializes the payload this candidate is owed: a thinking signature is + // rewritten for the upstream that will see it, and one that no upstream but the issuer + // can read is dropped rather than sent on. The id the client addressed does not travel — + // the provider re-stamps whatever it resolved upstream — and an alias' own rules apply to + // the body that is sent. + const asked = use.chatPayloadFor(facts['route.attempt']) as MessagesPayload; + const payload = { ...asked, model: candidate.model.id }; if (candidate.rules !== undefined) applyRulesToUpstreamMessages(payload, candidate.rules); const { model: _addressed, ...body } = payload; @@ -224,6 +229,9 @@ const callMessagesUpstream = defineStage< }); } + // This candidate answered, so it is the one a follow-up turn carrying our own state + // must come back to. + use.selectAffinity(candidate); const metered = meterMessages(result.events, identity); return move({ ...facts, @@ -267,9 +275,16 @@ const meterMessages = ( }; /** An upstream that reported nothing leaves no quantities at all, which is a different - * statement from reporting zero. */ -const billed = (usage: BillableUsage | undefined): UsageQuantities => - usage === undefined ? {} : tokenUsageFromBillableUsage(usage) as unknown as UsageQuantities; + * statement from reporting zero. + * + * A billed entity is keyed by billing metric, which is not the shape a protocol reports + * in — so the reading is converted here rather than cast. The service tier survives as far + * as `TokenUsage.tier` and no further: a billed entity is an identity and a bag of + * quantities, and the pricing selector the tier feeds has no seat there. */ +const billed = (usage: BillableUsage | undefined): UsageQuantities => { + const tokens = tokenUsageFromBillableUsage(usage); + return tokens === null ? {} : tokenUsageQuantities(tokens); +}; /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is From 1c89f49af44e8d18a88d53433ce19f9fd5600a27 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:17:16 +0800 Subject: [PATCH 012/156] feat(chat): assemble the Responses chain on the pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds `/v1/responses`' own wire as a pipeline, on the chain Chat Completions established: the edge that writes SSE frames or the one response object, settlement above the fork, candidate narrowing ordered by affinity, failover, and an ending that dials `callResponses`. Two things this family states that its siblings do not. A provider answers with the branch it actually ran, and one of them is not a stream — a compaction is a single envelope that carries its own counts — so it rides at the response key's value arm rather than at a key of its own or lowered into synthesized frames. And the client's stream is terminated by `[DONE]` here rather than by whatever the upstream's stream ended on, which is what the transport reads to know the turn is over; the ending stops reading at the turn's terminal event, and a stream that never states one ends the run with the sentence the replaced surface used. Scope is one wire and one transport. The WebSocket entry, `/v1/responses/compact`, the two translated wires, this family's interceptors and the stored-items membrane are each a step of their own and are named in the file header rather than implied by absence. Tests run the chain: frames written out, frames folded into one response, an envelope served as itself, a refusal answered in the upstream's own status and words, and a dial nobody answered failed over to the next candidate. --- .../chat/responses-pipeline_test.ts | 287 ++++++++++++++ .../src/data-plane/chat/responses/pipeline.ts | 369 ++++++++++++++++++ 2 files changed, 656 insertions(+) create mode 100644 packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts create mode 100644 packages/gateway/src/data-plane/chat/responses/pipeline.ts diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts new file mode 100644 index 0000000000..2463b08950 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -0,0 +1,287 @@ +// The Responses chain, run. Assembly is checked where every family's is; what is written +// down here is what only running it can say — that the edge terminates the client's stream +// on `[DONE]` however the upstream's ended, that it folds a stream into one response object +// when the client did not ask to stream, that a turn the upstream answered with one envelope +// instead of a stream is served as that envelope, that a refusal keeps the upstream's own +// status and words, and that a dial nobody answered is a value the fork can move past. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import type { CanonicalResponsesPayload, ResponsesCompactionResult, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import { directFetcher, type ModelCandidate, type ProviderResponsesResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +const candidate = ( + callResponses: (model: unknown, body: unknown, action: unknown) => Promise, + upstreamId = 'up_a', +): ModelCandidate => { + const endpoints = { responses: {} }; + return { + provider: { + upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callResponses }), + }, + model: stubInternalModel( + { id: 'responses-model', endpoints, providerModels: { [upstreamId]: stubProviderModel({ id: 'responses-model', endpoints }) } }, + upstreamId, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const delta = (text: string): ResponsesStreamEvent => ({ + type: 'response.output_text.delta', + sequence_number: 1, + item_id: 'msg_1', + output_index: 0, + content_index: 0, + delta: text, +}); + +const result = (text: string, usage?: ResponsesResult['usage']): ResponsesResult => ({ + id: 'resp_1', + object: 'response', + model: 'responses-model', + status: 'completed', + output: [{ + type: 'message', id: 'msg_1', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text, annotations: [] }], + }], + error: null, + incomplete_details: null, + ...(usage === undefined ? {} : { usage }), +}); + +const completed = (text: string, usage?: ResponsesResult['usage']): ResponsesStreamEvent => ({ + type: 'response.completed', + sequence_number: 2, + response: result(text, usage), +}); + +// The upstream's own stream, as the provider hands it up: typed frames, and the `done` frame +// its SSE transport ended on. +const stream = (...events: readonly ResponsesStreamEvent[]): ProviderResponsesResult => ({ + action: 'generate', + ok: true, + modelKey: 'responses-model-key', + headers: new Headers({ 'x-request-id': 'req-1', 'content-length': '99' }), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event }; + yield { type: 'done' as const }; + })(), +}); + +const payload = { + model: 'responses-model', + input: [{ type: 'message', role: 'user', content: 'hi' }], +} as unknown as CanonicalResponsesPayload; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * payload the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: CanonicalResponsesPayload = payload; + +const serve = async (wantsStream: boolean) => await run( + responsesServePipeline(payload), + move({ + 'ingress.http.headers': [] as readonly (readonly [string, string])[], + 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.responses.wantsStream': wantsStream, + 'request.chat.responses': payload, + 'serve.model': 'responses-model', + }) as never, + { + gateway: mockChatGatewayCtx({ wantsStream }), + background: () => {}, + rememberCandidates: () => {}, + rememberChatSelection: () => {}, + chatPayloadFor: () => affinityPayload, + selectAffinity: () => {}, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, +); + +const drain = async (rendered: unknown): Promise => { + const frames: SseFrame[] = []; + for await (const frame of rendered as AsyncIterable) frames.push(frame); + return frames; +}; + +beforeEach(() => { + affinityPayload = payload; + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the responses chain', () => { + // Affinity rewrites client-carried state for the upstream that will see it, so the body + // that goes out is the one it materialized rather than the one the client sent — and this + // chain dials the generate action, whatever else the protocol can ask an upstream for. + it('sends the payload affinity materialized for the candidate it dialled', async () => { + let sent: Record | undefined; + let asked: unknown; + affinityPayload = { + ...payload, + input: [{ type: 'message', role: 'user', content: 'rewritten' }], + } as CanonicalResponsesPayload; + resolves([candidate(async (_model, body, action) => { + sent = body as Record; + asked = action; + return stream(completed('hi')); + })]); + + await serve(false); + + expect(sent).toMatchObject({ input: [{ type: 'message', role: 'user', content: 'rewritten' }] }); + expect(sent).not.toHaveProperty('model'); + expect(asked).toBe('generate'); + }); + + // Every event goes out under its own SSE name, and the client's stream ends on the literal + // `[DONE]` the transport reads as the turn being over. + it('writes the frames out when the client asked to stream', async () => { + resolves([candidate(async () => stream(delta('he'), delta('llo'), completed('hello')))]); + + const { facts } = await serve(true); + const frames = await drain(facts['response.chat.responses.rendered']); + + expect(facts['response.http.status']).toBe(200); + expect(frames.map(frame => frame.event)).toEqual([ + 'response.output_text.delta', + 'response.output_text.delta', + 'response.completed', + undefined, + ]); + expect(frames.map(frame => frame.data)).toEqual([ + JSON.stringify(delta('he')), + JSON.stringify(delta('llo')), + JSON.stringify(completed('hello')), + '[DONE]', + ]); + }); + + // A stream's numbers arrive with its last chunk, so what the run hands up is a promise and + // the reading is taken off the frames the client itself drove. + it('bills what the terminal event reported, once the frames have run out', async () => { + resolves([candidate(async () => stream( + delta('hi'), + completed('hi', { input_tokens: 11, output_tokens: 7, total_tokens: 18 }), + ))]); + + const { facts } = await serve(true); + await drain(facts['response.chat.responses.rendered']); + const billable = await facts['response.chat.responses.streamedUsage']!; + + expect(billable).toEqual([expect.objectContaining({ + quantities: { input_tokens: '11', output_tokens: '7' }, + })]); + }); + + // The upstream speaks SSE whatever the client asked for, so a client that did not ask to + // stream is answered from the same frames — folded here rather than read a second time. + it('folds the frames into one response when the client did not', async () => { + resolves([candidate(async () => stream(delta('he'), delta('llo'), completed('hello')))]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.responses.rendered']).toMatchObject({ + id: 'resp_1', + status: 'completed', + output: [{ content: [{ type: 'output_text', text: 'hello' }] }], + }); + }); + + // A compaction is one envelope rather than a stream: the upstream ran the turn, charged for + // it, and stated the counts in the body. It rides at the same key the frames would have. + it('serves an upstream that answered with one envelope as that envelope', async () => { + const compaction: ResponsesCompactionResult = { + id: 'resp_compact_1', + object: 'response.compaction', + output: [{ + type: 'message', id: 'msg_c', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text: 'summary so far', annotations: [] }], + }], + usage: { input_tokens: 900, output_tokens: 40, total_tokens: 940 }, + }; + resolves([candidate(async () => ({ + action: 'compact', ok: true, modelKey: 'responses-model-key', result: compaction, + }))]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.responses.rendered']).toEqual(compaction); + expect(facts['response.chat.responses.streamedUsage']).toBeNull(); + expect(facts['response.usage.billable']).toEqual([expect.objectContaining({ + quantities: { input_tokens: '900', output_tokens: '40' }, + })]); + }); + + // Content-length would misdescribe a body this gateway serialized itself; a vendor trace + // is what a client and an operator both need to correlate a turn. + it('forwards the upstream headers a client may see and drops the ones it may not', async () => { + resolves([candidate(async () => stream(completed('hi')))]); + + const { facts } = await serve(true); + + expect(Object.fromEntries(facts['response.http.headers'])).toMatchObject({ 'x-request-id': 'req-1' }); + expect(Object.fromEntries(facts['response.http.headers'])).not.toHaveProperty('content-length'); + }); + + it('answers an upstream refusal with its own status and words', async () => { + resolves([candidate(async () => ({ + action: 'generate', + ok: false, + modelKey: 'responses-model-key', + response: new Response(JSON.stringify({ error: { message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + }))]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(429); + expect(facts['response.chat.responses.rendered']).toEqual({ error: { message: 'slow down' } }); + }); + + // A refused connection is an outcome the fork has to be able to see, not a fault that ends + // the run — so the second candidate is tried and its answer is the one served. + it('fails a dial that never connected over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); }, 'up_dead'), + candidate(async () => { tried.push('alive'); return stream(completed('hi')); }, 'up_alive'), + ]); + + const { facts } = await serve(false); + + expect(tried).toEqual(['dead', 'alive']); + expect(facts['response.http.status']).toBe(200); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts new file mode 100644 index 0000000000..9fdc059e80 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -0,0 +1,369 @@ +// Responses as a pipeline, on the chain Chat Completions established. +// +// emitResponses the edge: writes the answer in the shape the client asked for +// writeSettlement above the fork, so a run bills once however many wires it tried +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// callResponsesUpstream the ending: dials this candidate's wire +// +// One wire and one transport: `/v1/responses` over HTTP, dialled on a candidate's own +// Responses endpoint. Everything else this protocol owns is a step of its own, and is stated +// here rather than implied by its absence: +// +// - the WebSocket transport. A second entry against this protocol — its own framing, its +// own lifecycle — and `websocket.ts` still owns it. +// - `/v1/responses/compact`. A second operation over this protocol rather than another +// wire under this pipeline, so it stays on `responsesServe.compact`. The dial here asks +// for `generate`; what the ending answers with is the branch the provider says it ran, +// which is why the envelope a compaction is arrives somewhere rather than nowhere. +// - the translated wires, Responses via Messages and via Chat Completions. Each hands up +// the same `response.chat.responses` key, which is what will make them interchangeable +// with this one. +// - this family's interceptors, the server-tool shims among them. Still only in the +// interceptor form, so the array between the fork and the ending is empty rather than +// short. +// - the stored-items membrane: `previous_response_id` expansion, item hydration, and the +// snapshot the next turn reads. It wraps the chain rather than sitting in it, and +// `serve-prep.ts` and `client-output.ts` still hold it. +// +// One deliberate difference from `respond.ts`, shared with every other family on the +// pipeline: an upstream that refused is answered in its own words, with the status it sent, +// rather than being quoted back inside an envelope this gateway wrote. And one that is this +// family's own: a stream that runs out before its terminal event still ends the run with +// `RESPONSES_MISSING_TERMINAL_MESSAGE`, but nothing turns that into the mid-stream `error` +// and `response.failed` pair `respond.ts` wrote for a client already being streamed to. + +import { analyzeResponsesAffinity } from './affinity/ingress.ts'; +import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; +import type { BillableEntity } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { failover } from '../../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; +import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import type { ChatFacts } from '../facts.ts'; +import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { doneFrame, renderErrorEnvelope, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { + collectResponsesProtocolEventsToResult, + isResponsesTerminalEvent, + responsesProtocolFrameToSSEFrame, + RESPONSES_MISSING_TERMINAL_MESSAGE, + type CanonicalResponsesPayload, + type ResponsesStreamEvent, +} from '@floway-dev/protocols/responses'; +import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; + +/** `/v1/responses` prefers its own wire, then the translated Messages path, then the + * translated Chat Completions path. */ +export const responsesTarget = chatTargetPicker(['responses', 'messages', 'chat-completions']); + +/** What this family adds to the chat space. */ +export interface ResponsesFacts extends ChatFacts { + /** What the client is actually sent — an object when it asked for one, SSE frames when it + * asked to stream. The edge provides it, so a dump shows what the client received. */ + 'response.chat.responses.rendered': Record | AsyncIterable; + /** What the upstream will have reported once the frames run out, and `null` when nothing + * streamed. Settling from this is the epilogue's job, after the drain. */ + 'response.chat.responses.streamedUsage': Promise | null; +} + +type R = { [P in K]: ResponsesFacts[P] }; + +/** + * The outermost edge. What reaches it is usually a stream — the upstream speaks SSE whatever + * the client asked for — so what this decides is whether the client sees the frames or the + * one response object they add up to. + * + * Collecting is therefore the edge's own work and not a second reading of the upstream: the + * same frames that would have gone out are folded here instead, by the protocol's own + * reassembly, which is what makes a stream that stopped short of its terminal event say so + * rather than answer with half a response. + */ +const emitResponses = defineStage< + R<'ingress.chat.responses.wantsStream'>, + R<'ingress.chat.responses.wantsStream'>, + R<'ingress.chat.responses.wantsStream' | 'response.chat.responses' | 'response.http.headers'>, + R<'response.chat.responses.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitResponses', + through: { + request: { needs: ['ingress.chat.responses.wantsStream'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.responses', 'response.http.headers'], + consumes: ['response.chat.responses', 'response.http.headers'], + provides: ['response.chat.responses.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.chat.responses': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.http.status': answer.status, + }; + } + // A turn the upstream answered with one body rather than a stream — the compaction + // envelope — is already the object the client is owed, so there is nothing to fold. + if (answer.kind === 'value') { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move(answer.body as Record), + 'response.http.status': 200, + }; + } + + const frames = answer.frames as AsyncIterable>; + if (!back['ingress.chat.responses.wantsStream']) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move( + await collectResponsesProtocolEventsToResult(frames) as unknown as Record, + ), + 'response.http.status': 200, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move(renderSSE(frames)), + 'response.http.status': 200, + }; + }, +}); + +/** Every Responses event has an SSE form of its own, so the render is a straight map. What + * it adds is the terminator: the client's stream ends on the literal `[DONE]` payload + * whether or not the upstream's stream carried one, because that is what the transport + * reads to know the turn is over. + * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx?plain=1#L84 */ +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + // The upstream's own terminator is not the client's — the ending stops reading at the + // turn's terminal event, and one terminator is written below however the frames ended. + if (frame.type === 'done') continue; + yield responsesProtocolFrameToSSEFrame(frame); + } + yield responsesProtocolFrameToSSEFrame(doneFrame()); + })(), +}); + +/** + * The native wire. It dials Responses and provides the answer at this family's own response + * key — which is what will make it interchangeable with a translated chain: both hand up + * `response.chat.responses`, and the stage above cannot tell which ran. + */ +const callResponsesUpstream = defineStage< + R<'request.chat.responses' | 'route.attempt' | 'ingress.http.headers'>, + R<'response.chat.responses' | 'response.chat.responses.streamedUsage' + | 'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'callResponsesUpstream', + return: { + provides: [ + 'response.chat.responses', + 'response.chat.responses.streamedUsage', + 'response.usage.billable', + 'response.http.headers', + ], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + + // Affinity materializes the payload this candidate is owed: client-carried state — an + // encrypted reasoning blob, a compaction the upstream issued — is rewritten for the + // upstream that will see it, which is the whole reason a turn can be pinned at all. The + // id the client addressed does not travel — the provider re-stamps whatever it resolved + // upstream — and an alias' own rules apply to the body that is sent. + const asked = use.chatPayloadFor(facts['route.attempt']) as CanonicalResponsesPayload; + const payload = { ...asked, model: candidate.model.id }; + if (candidate.rules !== undefined) applyRulesToUpstreamResponses(payload, candidate.rules); + const { model: _addressed, ...body } = payload; + + let result; + try { + result = await candidate.provider.instance.callResponses( + providerModelOf(candidate), + body, + 'generate', + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.chat.responses': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.responses.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.headers': [], + }); + } + + const identity = telemetryModelIdentity(candidate, result.modelKey); + // An upstream that was called and reported nothing, which is a different statement from + // reporting zero. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + + if (!result.ok) { + const text = await result.response.text(); + use.log.warn('upstream refused', { status: result.response.status }); + let parsed: unknown; + try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } + return move({ + ...facts, + 'response.chat.responses': { + status: result.response.status, + message: text, + ...(parsed === undefined ? {} : { body: parsed }), + }, + 'response.chat.responses.streamedUsage': null, + 'response.usage.billable': called, + 'response.http.headers': [...result.response.headers], + }); + } + + // This candidate answered, so it is the one a follow-up turn carrying our own state must + // come back to. + use.selectAffinity(candidate); + + // A provider answers with the branch it actually ran, and one of them is not a stream: + // a compaction is a single envelope that states its own counts, so there is nothing to + // meter and nothing left to read. The fact space already carries a value beside a + // stream at every protocol's response key, so that is where it rides. + if (result.action === 'compact') { + return move({ + ...facts, + 'response.chat.responses': { kind: 'value' as const, body: result.result }, + 'response.chat.responses.streamedUsage': null, + 'response.usage.billable': [billed(identity, billableUsageFromResponsesResult(result.result) ?? undefined)], + 'response.http.headers': [], + }); + } + + const metered = meterResponses(result.events, identity); + return move({ + ...facts, + 'response.chat.responses': { kind: 'stream' as const, frames: metered.frames }, + 'response.chat.responses.streamedUsage': metered.billable, + 'response.usage.billable': called, + 'response.http.headers': [...(result.headers ?? new Headers())], + }); + }, +}); + +/** Reads the upstream's own usage off its own events as they pass, so the reading costs one + * pass and the client's stream is what drives it. Responses states its counts on the + * lifecycle envelopes, and only one carrying real counts replaces the running figure, so an + * envelope that states none cannot wipe a good reading. */ +const meterResponses = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, +): { readonly frames: AsyncIterable>; readonly billable: Promise } => { + let settle!: (billable: readonly BillableEntity[]) => void; + const billable = new Promise(resolve => { settle = resolve; }); + const generator = (async function* () { + let reported: BillableUsage | undefined; + try { + for await (const frame of source) { + if (frame.type !== 'event') { + yield frame; + continue; + } + const usage = billableUsageFromResponsesEvent(frame.event); + if (usage !== null) reported = usage; + yield frame; + // The turn is over, so there is nothing further to read. An upstream that holds the + // connection open past its terminal event would otherwise hold the client's stream + // open with it; returning here closes the read, which cancels the upstream. + if (isResponsesTerminalEvent(frame.event)) return; + } + // Frames ran out with no terminal event, which is a turn nobody can answer from: the + // response was never stated complete, incomplete or failed. + throw new Error(RESPONSES_MISSING_TERMINAL_MESSAGE); + } finally { + // Reached however the frames ended — the terminal event, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + settle([billed(identity, reported)]); + } + })(); + return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; +}; + +/** An upstream that reported nothing leaves no quantities at all, which is a different + * statement from reporting zero. + * + * A reading that did arrive is converted rather than cast: a billed entity is keyed by + * billing metric, which is not the shape a protocol reports in. The tier rides along + * because on this protocol it is not a quantity but a rate selector — `service_tier` states + * the tier the turn was actually served at, and that is the pricing entry it is billed + * under. */ +const billed = (identity: TelemetryModelIdentity, usage: BillableUsage | undefined): BillableEntity => { + if (usage === undefined) return { identity, quantities: {} }; + const measurement = tokenUsageMeasurement(tokenUsageFromBillableUsage(usage)); + return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; +}; + +/** A candidate that cannot serve *this* request is not a candidate — and what the client's + * own turn carries decides the order the rest are tried in, which is why the narrowing is + * built from the request rather than being a constant. */ +const narrowing = (payload: CanonicalResponsesPayload): ChatNarrowing> => ({ + source: 'responses', + canServe: candidate => responsesTarget.canServe(candidate.model.endpoints), + affinity: async gateway => await analyzeResponsesAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the /responses endpoint.`, + refuse: (status, message) => ({ 'response.chat.responses': { status, message } }), + refuses: ['response.chat.responses'], +}); + +export type ResponsesServeEntry = R< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.responses.wantsStream' + | 'request.chat.responses' | 'serve.model' +>; + +export type ResponsesServeExit = R< + 'response.chat.responses.rendered' | 'response.chat.responses.streamedUsage' + | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' +>; + +export const responsesServePipeline = (payload: CanonicalResponsesPayload): Pipeline => + compose('responsesServe', [ + emitResponses, + writeSettlement( + handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), + handedUp => (handedUp as { 'response.chat.responses.streamedUsage'?: unknown })['response.chat.responses.streamedUsage'] !== null, + ), + resolveChatCandidates(narrowing(payload)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), + owns: [], + }), + callResponsesUpstream, + ]); From a000a921be95513ad133f5985f4ca97484e6e8e5 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:20:47 +0800 Subject: [PATCH 013/156] fix(chat): write each protocol's own error envelope, and forward what the upstream said MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three families rendered a gateway-synthesized refusal through the OpenAI envelope. A Gemini client reads error.status — the Google-RPC name — and a Messages client reads a top-level type and a request_id; neither shape carries those, so both would have found the fields missing. The rule the envelope already encoded stays: an upstream that refused in its own words is handed on in them, because those words are already what its client reads. Only a refusal the gateway itself produced has no body to forward, and that is the one case where the *client's* protocol decides the shape. That distinction is now named once and each protocol supplies its own envelope. A turn whose carried state needs two upstreams at once answers 400, as every family did before the shared stage invented a 409 for it. The three chat chains assemble alongside the six that came before them. --- .../data-plane/chat/gemini-pipeline_test.ts | 2 +- .../data-plane/pipeline/assembly_test.ts | 4 ++++ .../chat/chat-completions/pipeline.ts | 2 +- .../src/data-plane/chat/gemini/pipeline.ts | 4 ++-- .../src/data-plane/chat/messages/errors.ts | 24 +++++++++++++++++++ .../src/data-plane/chat/messages/pipeline.ts | 5 ++-- .../gateway/src/data-plane/chat/stages.ts | 5 +++- .../protocols/src/common/error-envelope.ts | 16 ++++++++++++- packages/protocols/src/common/index.ts | 2 +- 9 files changed, 55 insertions(+), 9 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index a38b9eba7f..f0474efc1a 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -235,7 +235,7 @@ describe('the gemini pipeline', () => { // upstream actually returned and the words that upstream used, rather than a synthesized // envelope quoting its serialized body back as a message. expect(facts['response.http.status']).toBe(400); - expect(facts['response.chat.gemini.rendered']).toEqual({ error: { code: 400, message: '{"error":{"message":"no"}}', status: 'INVALID_ARGUMENT' } }); + expect(facts['response.chat.gemini.rendered']).toEqual({ error: { message: 'no' } }); // An upstream that was called and reported nothing, which is a different statement from // reporting zero. expect(facts['response.usage.billable']).toEqual([ diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 7131930d01..5848a013e1 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -9,6 +9,8 @@ import { describe, expect, it } from 'vitest'; import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/pipeline.ts'; import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; +import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; +import { messagesServePipeline } from '../../../src/data-plane/chat/messages/pipeline.ts'; import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; import { imagesServePipeline } from '../../../src/data-plane/images/pipeline.ts'; @@ -23,6 +25,8 @@ const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] ['audio transcription', () => audioTranscriptionServePipeline], ['alpha search', () => searchServePipeline({ kind: 'search' } as never)], ['chat completions', () => chatCompletionsServePipeline({ model: 'm', messages: [] } as never)], + ['messages', () => messagesServePipeline({ model: 'm', messages: [] } as never)], + ['gemini', () => geminiServePipeline({ model: 'm', contents: [] } as never)], ]; describe('every family assembles', () => { diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 9e53f6b976..5fd314e4c7 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -15,12 +15,12 @@ import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; import { billableUsageFromChatCompletionsEvent } from './usage.ts'; import type { UsageQuantities } from '../../../repo/types.ts'; +import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 877b4538de..e0f65aece7 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -54,7 +54,7 @@ import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '.. import { renderGeminiError } from './errors.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import type { BillableUsage, ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; +import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectGeminiProtocolEventsToResult, geminiProtocolFrameToSSEFrame, @@ -121,7 +121,7 @@ const emitGemini = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(renderGeminiError(answer.status, answer.message)), + 'response.chat.gemini.rendered': move(renderProtocolError(answer.body, () => renderGeminiError(answer.status, answer.message))), 'response.http.status': answer.status, }; } diff --git a/packages/gateway/src/data-plane/chat/messages/errors.ts b/packages/gateway/src/data-plane/chat/messages/errors.ts index 9aa147e0a3..9dccb76536 100644 --- a/packages/gateway/src/data-plane/chat/messages/errors.ts +++ b/packages/gateway/src/data-plane/chat/messages/errors.ts @@ -11,6 +11,30 @@ import type { TranslatorInputError } from '@floway-dev/translate'; // Byte-shape matches Anthropic-direct: `{type:'error', error:{type, message}, // request_id}` with `request_id` at the top level (alongside `error`, not // nested inside it) and key order load-bearing for byte-faithfulness. +/** What a Messages client is sent when a turn produced no content. The envelope is + * Anthropic's own — `type` at the top, the error's own `type` inside, and a `request_id` a + * caller quotes back in a support thread — so it is the protocol's shape rather than the + * gateway's, and an OpenAI-shaped one would carry none of those fields. */ +export const renderMessagesError = (status: number, message: string): Record => ({ + type: 'error', + error: { type: anthropicErrorTypeForStatus(status), message }, + request_id: generateAnthropicId('req'), +}); + +/** The type name each status carries in this protocol's envelope. */ +const anthropicErrorTypeForStatus = (status: number): string => { + switch (status) { + case 400: return 'invalid_request_error'; + case 401: return 'authentication_error'; + case 403: return 'permission_error'; + case 404: return 'not_found_error'; + case 413: return 'request_too_large'; + case 429: return 'rate_limit_error'; + case 529: return 'overloaded_error'; + default: return status >= 500 ? 'api_error' : 'invalid_request_error'; + } +}; + const anthropicErrorResult = ( status: number, type: string, diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index a5a2100c40..5fd897abc9 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -29,8 +29,9 @@ import type { ChatFacts } from '../facts.ts'; import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { renderMessagesError } from './errors.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; -import { renderErrorEnvelope, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectMessagesProtocolEventsToResult, messagesProtocolFrameToSSEFrame, @@ -94,7 +95,7 @@ const emitMessages = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.chat.messages.rendered': move(renderProtocolError(answer.body, () => renderMessagesError(answer.status, answer.message))), 'response.http.status': answer.status, }; } diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts index 842c454aab..94d9243333 100644 --- a/packages/gateway/src/data-plane/chat/stages.ts +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -92,7 +92,10 @@ export const resolveChatCandidates = (narrowing: ChatNar const viable = candidates.filter(candidate => narrowing.canServe(candidate)); const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return refuse(409, selection.message); + // A turn whose carried state needs two upstreams at once is a request the client can fix + // by not carrying it, which is what every family called this before: a 400, not a + // conflict with something the gateway holds. + if ('kind' in selection) return refuse(400, selection.message); if (selection.candidates.length === 0) { const missing = sawModel ? narrowing.unsupported(model) diff --git a/packages/protocols/src/common/error-envelope.ts b/packages/protocols/src/common/error-envelope.ts index c92fbd7408..73f3fe860f 100644 --- a/packages/protocols/src/common/error-envelope.ts +++ b/packages/protocols/src/common/error-envelope.ts @@ -17,5 +17,19 @@ export const upstreamErrorMessage = (body: unknown): string | undefined => { return typeof body.error.message === 'string' ? body.error.message : undefined; }; +/** + * The rule, with the envelope left to the protocol that writes it. + * + * An upstream that refused in its own words is handed on in them whatever protocol it spoke, + * because those words are already the shape its own client reads. Only a refusal the gateway + * itself produced has no body to forward, and that is the one case where which protocol the + * *client* speaks decides the shape. + */ +export const renderProtocolError = ( + upstreamBody: unknown, + envelope: () => Record, +): Record => isJsonObject(upstreamBody) ? upstreamBody : envelope(); + +/** The envelope the OpenAI-shaped protocols write. */ export const renderErrorEnvelope = (message: string, upstreamBody?: unknown): Record => - isJsonObject(upstreamBody) ? upstreamBody : { error: { message, type: 'api_error' } }; + renderProtocolError(upstreamBody, () => ({ error: { message, type: 'api_error' } })); diff --git a/packages/protocols/src/common/index.ts b/packages/protocols/src/common/index.ts index 2b7a1293d2..6248b39afa 100644 --- a/packages/protocols/src/common/index.ts +++ b/packages/protocols/src/common/index.ts @@ -13,6 +13,6 @@ export * from './sse.ts'; export * from './parse-sse.ts'; export * from './parse-events.ts'; -export { renderErrorEnvelope, upstreamErrorMessage } from './error-envelope.ts'; +export { renderErrorEnvelope, renderProtocolError, upstreamErrorMessage } from './error-envelope.ts'; export { isJsonObject, type JsonObject } from './json.ts'; export { captureExtras } from './reassemble-extras.ts'; From 368721547361ba02f83975d8d040e8f60d0d727b Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:23:02 +0800 Subject: [PATCH 014/156] feat(gateway): let a family name what it writes on an idle connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seam wrote an SSE comment as keepalive, which is invisible to any client by design — but Anthropic defines a ping event and its clients read one, so the two are not the same wire byte. A family that has its own idle frame now names it alongside the frames it is answering with, and Messages exports the one it writes so the route that serves that chain cannot forget it. --- packages/gateway/src/data-plane/chat/messages/pipeline.ts | 7 ++++++- packages/gateway/src/data-plane/pipeline/serve.ts | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index 5fd897abc9..c3fe328986 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -31,7 +31,7 @@ import { chatTargetPicker } from '../shared/target-picker.ts'; import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { renderMessagesError } from './errors.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; -import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { collectMessagesProtocolEventsToResult, messagesProtocolFrameToSSEFrame, @@ -299,6 +299,11 @@ const narrowing = (payload: MessagesPayload): ChatNarrowing = { [P in K]: GatewayFacts[P] }; @@ -121,6 +121,10 @@ export type Rendered = /** An answer that *is* a stream. The frames go out as they arrive, which is why nothing * above waits for them and why what they billed is settled afterwards. */ readonly frames: AsyncIterable; + /** What is written on an idle connection to keep it open. A comment is invisible to any + * client, but a protocol that defines its own idle event is read by clients that expect + * one — Anthropic's `ping` is a frame Claude Code sees — so the family names it. */ + readonly keepAlive?: SseWritableFrame; }; /** Which of the two an answer turned out to be. A family's rendered fact carries whichever @@ -173,7 +177,7 @@ export const serveThrough = async < return finalizeGatewayResponse(prologue.gateway, streamSSE(c, async stream => { try { await writeSSEFrames(stream, answer.frames, { - keepAlive: { frame: sseCommentFrame('keepalive') }, + keepAlive: { frame: answer.keepAlive ?? sseCommentFrame('keepalive') }, ...(prologue.gateway.downstreamAbortController === undefined ? {} : { downstreamAbortController: prologue.gateway.downstreamAbortController }), From 9b08dd1d908ac350089616f7e5191df67a2b3c67 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:53:27 +0800 Subject: [PATCH 015/156] fix(chat): send what the interceptors rewrote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The interceptors rewrite the request as a fact and the ending read the payload back from the resolver, so all three stages mutated a copy nobody sent. Deleting every one of them left the suite green. Affinity's per-candidate payload now enters the record below the fork, where there is a payload to speak of — each candidate has its own, and re-running the suffix is what produces the next. Everything between that stage and the dial rewrites a fact, and the ending sends the fact. Four comments claimed the fact space deletes the ctx.targetApi guards. The record retracts that: the claim rested on splitting the interceptor array by role, which was rejected — 「这里没有 source 和 target 之分!这是同一个数组!」 — and with one array the guards remain. What changed is that what they test is declared rather than ambient, which is visibility, not deletion. --- .../chat/chatcompletions-pipeline_test.ts | 29 +++++++++++++++++-- .../chat/chat-completions/pipeline.ts | 14 ++++----- packages/gateway/src/data-plane/chat/facts.ts | 19 ++++++------ .../gateway/src/data-plane/chat/handoff.ts | 7 +++-- .../src/data-plane/chat/interceptors.ts | 7 +++-- .../gateway/src/data-plane/chat/stages.ts | 28 ++++++++++++++++++ 6 files changed, 80 insertions(+), 24 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index ff2ffdbfcd..3cfae1f877 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -12,7 +12,7 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { directFetcher, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; +import { directFetcher, type FlagId, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -25,6 +25,7 @@ let live: readonly ModelCandidate[] = []; const candidate = ( callChatCompletions: (model: unknown, body: unknown) => Promise>, upstreamId = 'up_a', + flags: readonly FlagId[] = [], ): ModelCandidate => { const endpoints = { chatCompletions: {} }; return { @@ -34,7 +35,11 @@ const candidate = ( instance: stubProvider({ callChatCompletions }), }, model: stubInternalModel( - { id: 'chat-model', endpoints, providerModels: { [upstreamId]: stubProviderModel({ id: 'chat-model', endpoints }) } }, + { + id: 'chat-model', + endpoints, + providerModels: { [upstreamId]: stubProviderModel({ id: 'chat-model', endpoints, enabledFlags: new Set(flags) }) }, + }, upstreamId, ), fetcher: directFetcher, @@ -101,6 +106,26 @@ beforeEach(() => { }); describe('the chat completions chain', () => { + // The interceptors rewrite the request as a fact, so what they rewrite has to be what the + // ending sends. An ending that asked the resolver for the payload instead would read one no + // interceptor had touched, and every rewrite in the chain would go nowhere. + it('sends what the interceptors rewrote, not what the resolver materialized', async () => { + let sent: { messages: { role: string }[] } | undefined; + affinityPayload = { + ...payload, + messages: [{ role: 'system', content: 'be brief' }, { role: 'user', content: 'hi' }], + } as unknown as ChatCompletionsPayload; + resolves([candidate(async (_model, body) => { + sent = body as { messages: { role: string }[] }; + return stream(chunk('hi')); + }, 'up_a', ['rewrite-system-to-developer'])]); + + await serve(false); + + expect(sent!.messages.map(message => message.role)).toEqual(['developer', 'user']); + affinityPayload = payload; + }); + // Affinity rewrites client-carried state for the upstream that will see it, so the body // that goes out is the one it materialized rather than the one the client sent. it('sends the payload affinity materialized for the candidate it dialled', async () => { diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 5fd314e4c7..936a7e35a8 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -32,7 +32,7 @@ import { } from '../interceptors.ts'; import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; -import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import { chatCompletionsProtocolFrameToSSEFrame, @@ -174,12 +174,11 @@ const callChatCompletionsUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - // Affinity materializes the payload this candidate is owed: client-carried state is - // rewritten for the upstream that will see it, which is the whole reason a turn can be - // pinned at all. The id the client addressed does not travel — the provider re-stamps - // whatever it resolved upstream — and an alias' own rules apply to the body that is sent. - const asked = use.chatPayloadFor(facts['route.attempt']) as ChatCompletionsPayload; - const payload = { ...asked, model: candidate.model.id }; + // What the record holds by now: the payload affinity materialized for this candidate, + // as every stage between the fork and here has rewritten it. The id the client addressed + // does not travel — the provider re-stamps whatever it resolved upstream — and an alias' + // own rules apply to the body that is sent. + const payload = { ...facts['request.chat.chatCompletions'], model: candidate.model.id }; if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(payload, candidate.rules); const { model: _addressed, ...body } = payload; @@ -316,6 +315,7 @@ export const chatCompletionsServePipeline = (payload: ChatCompletionsPayload): P failed: handedUp => isFailure((handedUp as { 'response.chat.chatCompletions'?: unknown })['response.chat.chatCompletions']), owns: [], }), + materializeAttempt('request.chat.chatCompletions'), applyRoleCompatibilityToChatCompletions, disableReasoningOnForcedToolChoiceForChatCompletions, stripPromptCacheKeyForChatCompletions, diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts index 854d5a3f80..ca22f89e8a 100644 --- a/packages/gateway/src/data-plane/chat/facts.ts +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -3,11 +3,12 @@ // // Two things the key layout says that the old context object could not. Each protocol has // its own request key and its own response key, so a translation *consumes* the source and -// *provides* the target — and `compose` then refuses a chain that could re-enter its own -// protocol, which is `ctx.targetApi === ` deleted and made structural. And -// `ingress.*` is what the client asked for: it survives the switch to whatever protocol the -// upstream turned out to speak, which is why `wantsStream` is four disjoint keys rather -// than one shared one. +// *provides* the target, and which protocol a stage is looking at is a declared need rather +// than an ambient field on a mutable context. That is visibility and checkability, not the +// deletion of the guards themselves — with one interceptor array the guards remain, and the +// migration is an equivalent one. And `ingress.*` is what the client asked for: it survives +// the switch to whatever protocol the upstream turned out to speak, which is why +// `wantsStream` is four disjoint keys rather than one shared one. import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; @@ -27,10 +28,10 @@ export interface ChatFacts extends GatewayFacts { /** Which protocol the client spoke. Read by the stages that must know — a vendor * normalizer genuinely does — rather than reached for through an ambient field. * - * This is one half of what `ctx.targetApi === ` used to test. The other half, - * "do not run on a re-entered request", is not a question a stage asks any more: the - * translating stage consumed the source key, so a stage that needs it cannot be placed - * below, and `compose` says so at assembly. */ + * This is what `ctx.targetApi` was, read through `needs` rather than reached for. The + * guards that field carried are not deleted by saying it this way — there is one + * interceptor array, so a stage that must not run on a re-entered request still says so + * itself. What changed is that the thing it tests is declared. */ 'ingress.chat.sourceProtocol': ChatSourceProtocol; /** What the client asked for, per protocol and disjoint, because a translated request * must not inherit the target protocol's answer to a question the client never asked. */ diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index a6a33a4a1b..b00329d145 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -5,9 +5,10 @@ // target's response key and provides the source's. Two consequences follow from saying it // that way rather than carrying a `targetApi` field. // -// A chain cannot re-enter its own protocol. The source key is gone below this stage, so a -// stage that needs it cannot be placed there and `compose` says so at assembly — which is -// `ctx.targetApi === ` deleted and made structural rather than tested at runtime. +// The source key is gone below this stage, so a stage that needs it cannot be placed there +// and `compose` says so at assembly. That is a check on one chain's arrangement and not a +// proof that no chain can re-enter a protocol: a later handoff provides the key again, and +// nothing here counts hops — by ruling, because the graph does not loop. // // And nothing below knows a translation happened. The target chain sees its own protocol's // keys and only those, so the same chain serves a native request and a translated one. diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index 7b1ab104ff..a72fb51f84 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -7,9 +7,10 @@ // returns the record it hands on instead of assigning to `ctx.payload`, so a rewrite is a // spread and structural sharing is what makes it cheap — today's code already writes the // map conditionally, which is the convention that keeps a 49-message conversation costing -// three objects. And `ctx.targetApi !== ` is gone: array position says which chain a -// stage is in, and the translating stage above consumed the source key, so a re-entered -// request cannot reach a stage that needs it. +// three objects. And `ctx.targetApi` is a declared need rather than an ambient field — array +// position says which chain a stage is in, and what it reads about the protocol it is +// looking at is in the record. The guards themselves are not deleted by that: there is one +// interceptor array, so a rule that must not run on a re-entered request still says so. import type { Chat } from './facts.ts'; import { defineStage, move, transform } from '@floway-dev/pipeline'; diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts index 94d9243333..c0296ed7ae 100644 --- a/packages/gateway/src/data-plane/chat/stages.ts +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -113,6 +113,34 @@ export const resolveChatCandidates = (narrowing: ChatNar }, }); +/** + * Puts the payload this attempt is owed into the record. + * + * Affinity materializes one per candidate — client-carried state rewritten for the upstream + * that will see it — and it has to enter the record here rather than be read by the stage + * that dials, because everything between this stage and the dial rewrites the request as a + * fact. An ending that asked the resolver instead would be reading a payload no interceptor + * had touched, and the rewrites would go nowhere. + * + * It sits below the fork because there is no single payload above it: each candidate has its + * own, and re-running the suffix is what produces the next one. + */ +export const materializeAttempt = (requestKey: string) => defineStage< + Slice<'route.attempt'>, + Record, + Record, + Record, + ChatServices +>({ + name: `materializeAttempt:${requestKey}`, + through: { + request: { needs: ['route.attempt'], consumes: [], provides: [requestKey] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => + await next({ ...facts, [requestKey]: move(use.chatPayloadFor(facts['route.attempt'])) }), +}); + /** Everything about a candidate that is data. The live half — the provider instance, the * fetcher, the models cache — stays out of the record and is looked back up by the * resolver service at the moment of the call. */ From a87a234560bcbfdd66018958bf4b6a4111b9d264 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 02:57:47 +0800 Subject: [PATCH 016/156] fix(chat): keep the fields an OpenAI client reads on a refusal the gateway made MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A gateway-synthesized refusal reached the client as type: 'api_error' — which says the gateway broke rather than that the request did — and Responses lost the param and code that name which field was at fault and why routing failed. A protocol's envelope carries more than a status and a sentence, and only whoever refused knows those, so the refusal now carries what it would write and the edge renders it. The upstream's own body still wins wherever there is one. Which refusal it was travels with it: a turn that cannot be routed, a model no wire reaches, and a model no upstream has are three different things to say. --- .../chat/chatcompletions-pipeline_test.ts | 14 +++++++++++ .../chat/chat-completions/pipeline.ts | 23 ++++++++++++++++--- .../src/data-plane/chat/responses/pipeline.ts | 23 ++++++++++++++++--- .../gateway/src/data-plane/chat/stages.ts | 14 +++++++---- .../gateway/src/data-plane/pipeline/facts.ts | 5 ++++ 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index 3cfae1f877..b9faac7d68 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -211,6 +211,20 @@ describe('the chat completions chain', () => { expect(billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); }); + // A refusal the gateway made itself has no upstream body to forward, and that is the one + // case where the client's own protocol decides the shape. An OpenAI client reads + // error.type, and `api_error` would say the gateway broke rather than that the request did. + it('names its own refusal in the type an OpenAI client reads', async () => { + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [], sawModel: true, failedUpstreams: [] } as never); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.chatCompletions.rendered']).toEqual({ + error: { message: 'Model chat-model does not support the /chat/completions endpoint.', type: 'invalid_request_error' }, + }); + }); + // A refused connection is an outcome the fork has to be able to see, not a fault that ends // the run — so the second candidate is tried and its answer is the one served. it('fails a dial that never connected over to the next candidate', async () => { diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 936a7e35a8..c0d44a32f0 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -40,7 +40,7 @@ import { type ChatCompletionsPayload, type ChatCompletionsStreamEvent, } from '@floway-dev/protocols/chat-completions'; -import { renderErrorEnvelope, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; /** `/v1/chat/completions` prefers its own wire, then the translated Messages path, then the @@ -100,7 +100,10 @@ const emitChatCompletions = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.chatCompletions.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.chat.chatCompletions.rendered': move(renderProtocolError( + answer.body, + () => answer.envelope ?? { error: { message: answer.message, type: 'api_error' } }, + )), 'response.http.status': answer.status, }; } @@ -288,7 +291,21 @@ const narrowing = (payload: ChatCompletionsPayload): ChatNarrowing chatCompletionsTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeChatCompletionsAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /chat/completions endpoint.`, - refuse: (status, message) => ({ 'response.chat.chatCompletions': { status, message } }), + refuse: (status, message, reason) => ({ + 'response.chat.chatCompletions': { + status, + message, + // What an OpenAI client reads: the condition's own type, and for a turn whose carried + // state cannot be routed, the field at fault and the code that names it. + envelope: { + error: { + message, + type: 'invalid_request_error', + ...(reason === 'routing-unavailable' ? { param: 'input', code: 'responses_item_routing_unavailable' } : {}), + }, + }, + }, + }), refuses: ['response.chat.chatCompletions'], }); diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index 9fdc059e80..ea71192914 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -48,7 +48,7 @@ import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; -import { doneFrame, renderErrorEnvelope, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { doneFrame, renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, isResponsesTerminalEvent, @@ -113,7 +113,10 @@ const emitResponses = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.chat.responses.rendered': move(renderProtocolError( + answer.body, + () => answer.envelope ?? { error: { message: answer.message, type: 'api_error' } }, + )), 'response.http.status': answer.status, }; } @@ -339,7 +342,21 @@ const narrowing = (payload: CanonicalResponsesPayload): ChatNarrowing responsesTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeResponsesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /responses endpoint.`, - refuse: (status, message) => ({ 'response.chat.responses': { status, message } }), + refuse: (status, message, reason) => ({ + 'response.chat.responses': { + status, + message, + // What an OpenAI client reads: the condition's own type, and for a turn whose carried + // state cannot be routed, the field at fault and the code that names it. + envelope: { + error: { + message, + type: 'invalid_request_error', + ...(reason === 'routing-unavailable' ? { param: 'input', code: 'responses_item_routing_unavailable' } : {}), + }, + }, + }, + }), refuses: ['response.chat.responses'], }); diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts index c0296ed7ae..33bdee5eca 100644 --- a/packages/gateway/src/data-plane/chat/stages.ts +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -32,6 +32,10 @@ export interface ChatServices extends GatewayServices { readonly selectAffinity: (candidate: ModelCandidate) => void; } +/** Why the gateway refused before it reached an upstream. A protocol's envelope names the + * condition in its own words, and this is what it names. */ +export type ChatRefusal = 'routing-unavailable' | 'model-unsupported' | 'model-missing'; + /** How a source protocol narrows and orders the candidates that could serve it. */ export interface ChatNarrowing { readonly source: ChatSourceProtocol; @@ -43,7 +47,7 @@ export interface ChatNarrowing { * upstream that issued it, so this runs before any candidate is tried. */ readonly affinity: (gateway: ChatGatewayCtx) => Promise>; readonly unsupported: (model: string) => string; - readonly refuse: (status: number, message: string) => Refusal; + readonly refuse: (status: number, message: string, reason: ChatRefusal) => Refusal; readonly refuses: readonly (keyof Refusal)[]; } @@ -82,12 +86,12 @@ export const resolveChatCandidates = (narrowing: ChatNar // An empty billed set is what "we did not call an upstream" looks like, and an empty // header list is the same statement on the other key. - const refuse = (status: number, message: string) => + const refuse = (status: number, message: string, reason: ChatRefusal) => move({ ...facts, 'response.usage.billable': [], 'response.http.headers': [], - ...narrowing.refuse(status, message), + ...narrowing.refuse(status, message, reason), }); const viable = candidates.filter(candidate => narrowing.canServe(candidate)); @@ -95,12 +99,12 @@ export const resolveChatCandidates = (narrowing: ChatNar // A turn whose carried state needs two upstreams at once is a request the client can fix // by not carrying it, which is what every family called this before: a 400, not a // conflict with something the gateway holds. - if ('kind' in selection) return refuse(400, selection.message); + if ('kind' in selection) return refuse(400, selection.message, 'routing-unavailable'); if (selection.candidates.length === 0) { const missing = sawModel ? narrowing.unsupported(model) : `Model ${model} is not available on any configured upstream.`; - return refuse(sawModel ? 400 : 404, appendFailedUpstreams(missing, failedUpstreams)); + return refuse(sawModel ? 400 : 404, appendFailedUpstreams(missing, failedUpstreams), sawModel ? 'model-unsupported' : 'model-missing'); } // The live half stays with the resolver; only selectors travel. The payload a candidate diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index ad65f85777..77c6a72f12 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -45,6 +45,11 @@ export interface Failure { /** The upstream's own body, when there was one. A client is not owed the upstream's * exact bytes, but a dump reader is owed what actually came back. */ readonly body?: unknown; + /** What the gateway itself would write, when the refusal is its own. A protocol's envelope + * carries more than a status and a sentence — which field was at fault, which code names + * the condition — and only whoever refused knows those, so it is written where the refusal + * is made rather than derived from the status where it is rendered. */ + readonly envelope?: Record; } export const isFailure = (value: unknown): value is Failure => From 24f3406c2fef90a4139471e12513e93bc6807f8e Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 03:01:03 +0800 Subject: [PATCH 017/156] fix(chat): time the first token, and price by what the rate depends on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Time to first token was written in one place no stage reaches, so every chat run took the neutral branch and recorded no TTFT at all. It is measured where the token is — the only place that can tell a generated frame from the envelope around it. A rate can depend on the service tier and on how much input there was. Both are selector coordinates rather than quantities, and dropping them left the pricer with an empty selector, so neither axis could ever apply. The reading now goes through the same measurement the other families use and carries both. --- .../chat/chatcompletions-pipeline_test.ts | 22 ++++++++++++-- .../chat/chat-completions/pipeline.ts | 30 ++++++++++++------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index b9faac7d68..f12ab16ef1 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -72,7 +72,9 @@ const payload = { model: 'chat-model', messages: [{ role: 'user', content: 'hi' * payload the client sent, which is the point: carried state is rewritten per candidate. */ let affinityPayload: ChatCompletionsPayload = payload; -const serve = async (wantsStream: boolean) => await run( +const serve = async (wantsStream: boolean) => await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream); + +const serveWith = async (gateway: ReturnType, wantsStream: boolean) => await run( chatCompletionsServePipeline(payload), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], @@ -83,7 +85,7 @@ const serve = async (wantsStream: boolean) => await run( 'serve.model': 'chat-model', }) as never, { - gateway: mockChatGatewayCtx({ wantsStream }), + gateway, background: () => {}, rememberCandidates: () => {}, rememberChatSelection: () => {}, @@ -209,6 +211,22 @@ describe('the chat completions chain', () => { const billable = await facts['response.chat.chatCompletions.streamedUsage']!; expect(billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); + // A rate can depend on the tier and on how much input there was; both are selector + // coordinates, so they travel as pricing facts rather than as quantities. + expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 11 }); + }); + + // Time to first token is measured where the token is — an envelope frame is not one, and a + // run that never stamps it is recorded as having produced nothing to time. + it('stamps the first frame that carried a generated token', async () => { + resolves([candidate(async () => stream(chunk('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + expect(gateway.attempt.firstOutputTokenAt).toBeNull(); + for await (const _frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { /* drain */ } + + expect(gateway.attempt.firstOutputTokenAt).toBeTypeOf('number'); }); // A refusal the gateway made itself has no upstream body to forward, and that is the one diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index c0d44a32f0..50a288c80b 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -14,14 +14,12 @@ import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; import { billableUsageFromChatCompletionsEvent } from './usage.ts'; -import type { UsageQuantities } from '../../../repo/types.ts'; -import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; +import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; @@ -31,6 +29,7 @@ import { stripPromptCacheKeyForChatCompletions, } from '../interceptors.ts'; import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; +import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; @@ -235,7 +234,7 @@ const callChatCompletionsUpstream = defineStage< // This candidate answered, so it is the one a follow-up turn carrying our own state // must come back to. use.selectAffinity(candidate); - const metered = meterChatCompletions(result.events, identity); + const metered = meterChatCompletions(result.events, identity, use.gateway.attempt); return move({ ...facts, 'response.chat.chatCompletions': { kind: 'stream' as const, frames: metered.frames }, @@ -252,6 +251,7 @@ const callChatCompletionsUpstream = defineStage< const meterChatCompletions = ( source: AsyncIterable>, identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, ): { readonly frames: AsyncIterable>; readonly billable: Promise } => { let settle!: (billable: readonly BillableEntity[]) => void; const billable = new Promise(resolve => { settle = resolve; }); @@ -259,6 +259,13 @@ const meterChatCompletions = ( let reported: BillableUsage | undefined; try { for await (const frame of source) { + // Time to first token is measured where the token is, which is the only place that + // knows a frame carries generated content rather than the envelope around it. + // Time to first token is measured where the token is, which is the only place that + // knows a frame carries generated content rather than the envelope around it. + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'chat-completions')) { + attempt.firstOutputTokenAt = performance.now(); + } if (frame.type === 'event') { const usage = billableUsageFromChatCompletionsEvent(frame.event); if (usage !== null) reported = usage; @@ -269,18 +276,21 @@ const meterChatCompletions = ( // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle([{ identity, quantities: billed(reported) }]); + settle([billedEntity(reported, identity)]); } })(); return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; }; -/** An upstream that reported nothing leaves no quantities at all, which is a different - * statement from reporting zero. The two shapes are not interchangeable: what the upstream - * reported is per token category, and what is billed is per metric name. */ -const billed = (usage: BillableUsage | undefined): UsageQuantities => { +/** What one attempt is billable for. An upstream that reported nothing leaves no quantities + * at all, which is a different statement from reporting zero — and a rate can depend on the + * service tier and on how much input there was, so both travel as pricing facts rather than + * being folded into the quantities. */ +const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { const tokens = tokenUsageFromBillableUsage(usage); - return tokens === null ? {} : tokenUsageQuantities(tokens); + if (tokens === null) return { identity, quantities: {} }; + const measurement = tokenUsageMeasurement(tokens); + return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; }; /** A candidate that cannot serve *this* request is not a candidate — and what the client's From 4293894beb9ce97b0d7e1966df91ee404dc2d523 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 03:07:17 +0800 Subject: [PATCH 018/156] refactor(chat): say what the chains do, and stop declaring what nothing reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared stages' header described a dispatch stage that picks a wire. No such stage was built — every chain dials its own protocol — so what it now describes is the payload entering the record below the fork, and what will decide a wire once the translated ones land. The selector is one statement about a candidate and was written twice; the chat resolver uses the definition the shared one already had. ChatNarrowing.source was declared and never read. A handoff declared it needs the upstream's headers and then supplied [] when they were absent, which would have hidden the assembly error that absence is. --- .../chat/chat-completions/pipeline.ts | 1 - .../gateway/src/data-plane/chat/handoff.ts | 2 +- .../gateway/src/data-plane/chat/stages.ts | 28 +++++++------------ .../gateway/src/data-plane/pipeline/stages.ts | 2 +- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 50a288c80b..423fed40aa 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -297,7 +297,6 @@ const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModel * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ const narrowing = (payload: ChatCompletionsPayload): ChatNarrowing> => ({ - source: 'chatCompletions', canServe: candidate => chatCompletionsTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeChatCompletionsAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /chat/completions endpoint.`, diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index b00329d145..7d1ae7daa6 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -78,7 +78,7 @@ export const handOff = ( // is read as text and parsed only for whoever wants the object. const rewritten = trip.apiError?.({ status: answer.status, - headers: new Headers(((up['response.http.headers'] ?? []) as readonly (readonly [string, string])[]).map(([name, value]): [string, string] => [name, value])), + headers: new Headers((up['response.http.headers'] as readonly (readonly [string, string])[]).map(([name, value]): [string, string] => [name, value])), body: new TextEncoder().encode(answer.message), }); if (rewritten === undefined) return { ...up, [handoff.from.response]: answer } as never; diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts index 33bdee5eca..f88a289084 100644 --- a/packages/gateway/src/data-plane/chat/stages.ts +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -2,24 +2,26 @@ // // Chat differs from the families in PR 2 in two ways, and both are here rather than in each // family. Candidates are ordered by affinity before they are tried, because client-carried -// state pins a turn to the upstream that issued it. And the last stage picks a *wire*: a -// candidate that serves this request may speak a different protocol than the client did, so -// what follows the fork is decided per candidate rather than at assembly. +// state pins a turn to the upstream that issued it. And the payload a candidate is dialled +// with is that candidate's own, because affinity rewrites client-carried state for the +// upstream that will see it — so it enters the record below the fork, where there is one +// payload to speak of, rather than above it where there are as many as there are candidates. // -// The second is why dispatch is an `into` stage. Only a pipeline's last stage may name what -// comes next, and failover re-runs the whole suffix — including that last stage — so the -// next candidate re-picks its own wire with no mechanism of its own. +// When the translated wires land, which wire a candidate is dialled on will be decided per +// candidate too, by the chain's last stage handing into the target's. Failover re-runs the +// whole suffix including that stage, so the next candidate re-picks with no mechanism of its +// own. Every chain here dials its own protocol, so nothing picks yet. -import type { ChatSourceProtocol } from './facts.ts'; import type { AttemptSelector, GatewayFacts } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; +import { selectorFor } from '../pipeline/stages.ts'; import { enumerateModelCandidates } from '../providers/resolution.ts'; import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; import { selectAffinityCandidates } from './shared/affinity/index.ts'; import type { AffinityRequestAnalysis } from './shared/affinity/selection.ts'; import type { ChatGatewayCtx } from './shared/gateway-ctx.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import { providerModelOf, type ModelCandidate } from '@floway-dev/provider'; +import type { ModelCandidate } from '@floway-dev/provider'; type Slice = { [P in K]: GatewayFacts[P] }; @@ -38,7 +40,6 @@ export type ChatRefusal = 'routing-unavailable' | 'model-unsupported' | 'model-m /** How a source protocol narrows and orders the candidates that could serve it. */ export interface ChatNarrowing { - readonly source: ChatSourceProtocol; /** Which upstream wires this source prefers, in order. A candidate none of whose endpoints * appear here cannot serve the request whatever else it offers. */ readonly canServe: (candidate: ModelCandidate) => boolean; @@ -144,12 +145,3 @@ export const materializeAttempt = (requestKey: string) => defineStage< execute: async (facts, next, use) => await next({ ...facts, [requestKey]: move(use.chatPayloadFor(facts['route.attempt'])) }), }); - -/** Everything about a candidate that is data. The live half — the provider instance, the - * fetcher, the models cache — stays out of the record and is looked back up by the - * resolver service at the moment of the call. */ -const selectorFor = (candidate: ModelCandidate): AttemptSelector => ({ - upstreamId: candidate.provider.upstreamId, - modelId: candidate.model.id, - flags: [...providerModelOf(candidate).enabledFlags], -}); diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts index c370adedcc..7a502479be 100644 --- a/packages/gateway/src/data-plane/pipeline/stages.ts +++ b/packages/gateway/src/data-plane/pipeline/stages.ts @@ -25,7 +25,7 @@ type Slice = { [P in K]: GatewayFacts[P] }; /** Everything about a candidate that is data. The live half — the provider instance, the * fetcher, the models cache — stays out of the record and is looked back up by the * resolver service at the moment of the call. */ -const selectorFor = (candidate: ModelCandidate): AttemptSelector => ({ +export const selectorFor = (candidate: ModelCandidate): AttemptSelector => ({ upstreamId: candidate.provider.upstreamId, modelId: candidate.model.id, flags: [...providerModelOf(candidate).enabledFlags], From d07fef24f4072ffc5f0e97f37252147ec500484a Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 03:16:36 +0800 Subject: [PATCH 019/156] fix(chat): send, time, bill and end a Messages turn as its chain states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ending read its payload back from the resolver while everything between the fork and the dial rewrites the request as a fact, so a rewrite would have gone nowhere. Affinity's per-candidate payload now enters the record below the fork, where there is one payload to speak of, and the ending sends the fact. Time to first token was written in one place no stage on this chain reaches, so every Messages run took the neutral branch and recorded no TTFT at all. It is measured where the token is — the only place that can tell a generated frame from the envelope around it. A rate can depend on the service tier and on how much input there was. Both are selector coordinates rather than quantities, and dropping them left the pricer with an empty selector, so neither axis could ever apply. The reading now goes through the same measurement the other families use and carries both. A stream that ran out before message_stop was served as a whole answer, and frames an upstream wrote after it went out to the client. The meter now stops at the terminal event and fails a stream that never reached one, which is what the protocol's own reassembly already did for the folded shape alone. ChatNarrowing.source goes with the key it was declared under. --- .../data-plane/chat/messages-pipeline_test.ts | 92 ++++++++++++++++--- .../src/data-plane/chat/messages/pipeline.ts | 65 ++++++++----- 2 files changed, 122 insertions(+), 35 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts index a3975688c9..0b55dfd2f4 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts @@ -14,7 +14,7 @@ import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; +import { MESSAGES_MISSING_TERMINAL_MESSAGE, type MessagesPayload, type MessagesStreamEvent } from '@floway-dev/protocols/messages'; import { directFetcher, type MessagesUpstreamCallOptions, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -75,6 +75,11 @@ const messageStop = { type: 'message_stop' }; const turn = (...text: readonly string[]): readonly unknown[] => [messageStart, blockStart, ...text.map(textDelta), blockStop, messageDelta, messageStop]; +/** The same turn with its terminal event cut off, which is what an upstream that died + * mid-answer leaves behind. */ +const truncatedTurn = (...text: readonly string[]): readonly unknown[] => + turn(...text).filter(event => event !== messageStop); + const stream = (events: readonly unknown[]): ProviderStreamResult => ({ ok: true, modelKey: 'claude-model-key', @@ -91,7 +96,18 @@ const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'use * payload the client sent, which is the point: carried state is rewritten per candidate. */ let affinityPayload: MessagesPayload = payload; -const serve = async (wantsStream: boolean, headers: readonly (readonly [string, string])[] = []) => await run( +/** How many times the chain asked the resolver for that payload. One stage owns the reading; + * anything below it reads the record. */ +let asked = 0; + +const serve = async (wantsStream: boolean, headers: readonly (readonly [string, string])[] = []) => + await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream, headers); + +const serveWith = async ( + gateway: ReturnType, + wantsStream: boolean, + headers: readonly (readonly [string, string])[] = [], +) => await run( messagesServePipeline(payload), move({ 'ingress.http.headers': headers, @@ -101,11 +117,11 @@ const serve = async (wantsStream: boolean, headers: readonly (readonly [string, 'serve.model': 'claude-model', }) as never, { - gateway: mockChatGatewayCtx({ wantsStream }), + gateway, background: () => {}, rememberCandidates: () => {}, rememberChatSelection: () => {}, - chatPayloadFor: () => affinityPayload, + chatPayloadFor: () => { asked += 1; return affinityPayload; }, selectAffinity: () => {}, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); @@ -124,6 +140,7 @@ const collect = async (rendered: unknown): Promise => { beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); affinityPayload = payload; + asked = 0; initRepo({ usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, @@ -133,8 +150,11 @@ beforeEach(() => { describe('the messages chain', () => { // Affinity rewrites client-carried state — a thinking signature only its issuer can read — // for the upstream that will see it, so the body that goes out is the one it materialized - // rather than the one the client sent. - it('sends the payload affinity materialized for the candidate it dialled', async () => { + // rather than the one the client sent. It enters the record below the fork because + // everything between there and the dial rewrites the request as a fact, and one stage owns + // that reading: an ending that asked the resolver again would send a payload no stage in + // the chain had touched. + it('puts the payload this attempt is owed into the record, and sends that', async () => { let sent: Record | undefined; affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as MessagesPayload; resolves([candidate(async (_model, body) => { @@ -142,8 +162,14 @@ describe('the messages chain', () => { return stream(turn('hi')); })]); - await serve(false); + const { facts } = await serve(false); + // The exit contract names what the run answers with, and the request the chain sent is + // not that — but the record carries every key in flight, so what was sent is still there + // to read. + expect((facts as Record)['request.chat.messages']).toBe(affinityPayload); + expect(asked).toBe(1); + // The id the client addressed does not travel: the provider re-stamps what it resolved. expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); expect(sent).not.toHaveProperty('model'); }); @@ -264,12 +290,54 @@ describe('the messages chain', () => { ]); await collect(facts['response.chat.messages.rendered']); + const billable = await facts['response.chat.messages.streamedUsage']!; + + expect(billable[0]).toMatchObject({ + identity: { model: 'claude-model', upstream: 'up_a', modelKey: 'claude-model-key', pricing: null }, + quantities: { input_tokens: '5', input_cache_read_tokens: '2', output_tokens: '7' }, + }); + // A rate can depend on the tier and on how much input there was; both are selector + // coordinates, so they travel as pricing facts rather than as quantities. + expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 7 }); + }); + + // Time to first token is measured where the token is — `message_start` and + // `content_block_start` are the envelope around the answer, not the answer, and a run that + // never stamps it is recorded as having produced nothing to time. + it('stamps the first frame that carried a generated token', async () => { + resolves([candidate(async () => stream(turn('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + expect(gateway.attempt.firstOutputTokenAt).toBeNull(); + await collect(facts['response.chat.messages.rendered']); + + expect(gateway.attempt.firstOutputTokenAt).toBeTypeOf('number'); + }); + + // A turn is over at `message_stop`, so what an upstream writes after it is not part of the + // answer and the client is not shown it. Reading stops there, which also closes the + // upstream rather than leaving the client's stream open behind a connection nobody reads. + it('stops reading at the terminal event', async () => { + resolves([candidate(async () => stream([...turn('hi'), textDelta(' and more')]))]); + + const { facts } = await serve(true); + const frames = await collect(facts['response.chat.messages.rendered']); - expect(await facts['response.chat.messages.streamedUsage']).toEqual([ - { - identity: { model: 'claude-model', upstream: 'up_a', modelKey: 'claude-model-key', pricing: null }, - quantities: { input_tokens: '5', input_cache_read_tokens: '2', output_tokens: '7' }, - }, + expect(frames.map(frame => frame.event)).toEqual([ + 'message_start', 'content_block_start', 'content_block_delta', + 'content_block_stop', 'message_delta', 'message_stop', ]); }); + + // A stream that ran out before `message_stop` is a turn nobody can answer from: the message + // was never stopped and never failed, so serving what did arrive would report a truncated + // answer as a whole one. + it('fails a stream that ended without its terminal event', async () => { + resolves([candidate(async () => stream(truncatedTurn('hi')))]); + + const { facts } = await serve(true); + + await expect(collect(facts['response.chat.messages.rendered'])).rejects.toThrow(MESSAGES_MISSING_TERMINAL_MESSAGE); + }); }); diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index c3fe328986..c5703d8373 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -4,6 +4,7 @@ // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate +// materializeAttempt puts the payload this candidate is owed into the record // callMessagesUpstream the ending: dials this candidate's wire // // Only the native wire is here. The translated ones — Messages via Responses and Messages @@ -11,30 +12,30 @@ // make them interchangeable with this one, and each is a step of its own. So is // `/v1/messages/count_tokens`: a second operation over this protocol rather than another // wire under this pipeline. This family's own interceptors are not stages yet either, so -// the array between the fork and the ending is empty rather than short. +// the array between the materialized payload and the ending is empty rather than short. import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { createMessagesBillableUsageReader } from './usage.ts'; -import type { UsageQuantities } from '../../../repo/types.ts'; -import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; +import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; +import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; -import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { renderMessagesError } from './errors.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import { renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { collectMessagesProtocolEventsToResult, messagesProtocolFrameToSSEFrame, + MESSAGES_MISSING_TERMINAL_MESSAGE, parseAnthropicBetaHeader, type MessagesPayload, type MessagesStreamEvent, @@ -64,8 +65,8 @@ type M = { [P in K]: MessagesFacts[P] }; * * Collecting is therefore the edge's own work and not a second reading of the upstream: the * same frames that would have gone out are folded here instead, by the protocol's own - * reassembly, which is what makes a stream that stopped short of `message_stop` say so - * rather than answer with half a message. + * reassembly. Neither shape can answer with half a message: the ending fails a stream that + * ran out before `message_stop`, whichever of the two the client asked for. */ const emitMessages = defineStage< M<'ingress.chat.messages.wantsStream'>, @@ -167,13 +168,13 @@ const callMessagesUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - // Affinity materializes the payload this candidate is owed: a thinking signature is + // What the record holds by now: the payload affinity materialized for this candidate, as + // every stage between the fork and here has rewritten it. A thinking signature was // rewritten for the upstream that will see it, and one that no upstream but the issuer - // can read is dropped rather than sent on. The id the client addressed does not travel — + // can read was dropped rather than sent on. The id the client addressed does not travel — // the provider re-stamps whatever it resolved upstream — and an alias' own rules apply to // the body that is sent. - const asked = use.chatPayloadFor(facts['route.attempt']) as MessagesPayload; - const payload = { ...asked, model: candidate.model.id }; + const payload = { ...facts['request.chat.messages'], model: candidate.model.id }; if (candidate.rules !== undefined) applyRulesToUpstreamMessages(payload, candidate.rules); const { model: _addressed, ...body } = payload; @@ -233,7 +234,7 @@ const callMessagesUpstream = defineStage< // This candidate answered, so it is the one a follow-up turn carrying our own state // must come back to. use.selectAffinity(candidate); - const metered = meterMessages(result.events, identity); + const metered = meterMessages(result.events, identity, use.gateway.attempt); return move({ ...facts, 'response.chat.messages': { kind: 'stream' as const, frames: metered.frames }, @@ -251,6 +252,7 @@ const callMessagesUpstream = defineStage< const meterMessages = ( source: AsyncIterable>, identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, ): { readonly frames: AsyncIterable>; readonly billable: Promise } => { let settle!: (billable: readonly BillableEntity[]) => void; const billable = new Promise(resolve => { settle = resolve; }); @@ -259,39 +261,55 @@ const meterMessages = ( let reported: BillableUsage | undefined; try { for await (const frame of source) { + // Time to first token is measured where the token is, which is the only place that + // knows a frame carries generated content rather than the envelope around it. + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'messages')) { + attempt.firstOutputTokenAt = performance.now(); + } if (frame.type === 'event') { const usage = readBillableUsage(frame.event); if (usage !== null) reported = usage; } yield frame; + // The turn is over, so there is nothing further to read. An upstream that holds the + // connection open past `message_stop` would otherwise hold the client's stream open + // with it; returning here closes the read, which cancels the upstream. + if (isMessagesTerminalFrame(frame)) return; } + // Frames ran out with no terminal event, which is a turn nobody can answer from: the + // message was never stopped and never failed. + throw new Error(MESSAGES_MISSING_TERMINAL_MESSAGE); } finally { // Reached however the frames ended — the terminal event, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle([{ identity, quantities: billed(reported) }]); + settle([billedEntity(reported, identity)]); } })(); return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; }; -/** An upstream that reported nothing leaves no quantities at all, which is a different - * statement from reporting zero. - * - * A billed entity is keyed by billing metric, which is not the shape a protocol reports - * in — so the reading is converted here rather than cast. The service tier survives as far - * as `TokenUsage.tier` and no further: a billed entity is an identity and a bag of - * quantities, and the pricing selector the tier feeds has no seat there. */ -const billed = (usage: BillableUsage | undefined): UsageQuantities => { +/** What ends a Messages turn. Anthropic's own stream terminator is an event rather than a + * transport sentinel, and a stream that failed mid-turn says so with `error` in place of the + * `message_stop` that will now never come. */ +const isMessagesTerminalFrame = (frame: ProtocolFrame): boolean => + frame.type === 'event' && (frame.event.type === 'message_stop' || frame.event.type === 'error'); + +/** What one attempt is billable for. An upstream that reported nothing leaves no quantities + * at all, which is a different statement from reporting zero — and a rate can depend on the + * service tier and on how much input there was, so both travel as pricing facts rather than + * being folded into the quantities. */ +const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { const tokens = tokenUsageFromBillableUsage(usage); - return tokens === null ? {} : tokenUsageQuantities(tokens); + if (tokens === null) return { identity, quantities: {} }; + const measurement = tokenUsageMeasurement(tokens); + return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; }; /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ const narrowing = (payload: MessagesPayload): ChatNarrowing> => ({ - source: 'messages', canServe: candidate => messagesTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeMessagesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /messages endpoint.`, @@ -326,5 +344,6 @@ export const messagesServePipeline = (payload: MessagesPayload): Pipeline isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages']), owns: [], }), + materializeAttempt('request.chat.messages'), callMessagesUpstream, ]); From 376bf324f0877191afab548caa233758a026d71d Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 03:22:49 +0800 Subject: [PATCH 020/156] fix(chat): send what the record holds on the Responses chain, and time its first token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ending read its payload back from the resolver while everything between the fork and the dial rewrites the request as a fact, so a rewrite would have gone nowhere. Affinity's per-candidate payload now enters the record below the fork, where there is one payload to speak of, and the ending sends the fact. Time to first token was written in one place no stage on this chain reaches, so every Responses run took the neutral branch and recorded no TTFT at all. It is measured where the token is — the only place that can tell a generated frame from the envelope around it. The two readings this chain already had right — pricing facts beside the quantities, and a stream that fails rather than answering short — were carried by no test, which is how the other chains came to disagree with them. They have one each now. ChatNarrowing.source goes with the key it was declared under. --- .../chat/responses-pipeline_test.ts | 80 ++++++++++++++++--- .../src/data-plane/chat/responses/pipeline.ts | 28 +++++-- 2 files changed, 89 insertions(+), 19 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index 2463b08950..c2bfbf5efc 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -13,7 +13,7 @@ import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import type { CanonicalResponsesPayload, ResponsesCompactionResult, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import { RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type ResponsesCompactionResult, type ResponsesResult, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; import { directFetcher, type ModelCandidate, type ProviderResponsesResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -99,7 +99,13 @@ const payload = { * payload the client sent, which is the point: carried state is rewritten per candidate. */ let affinityPayload: CanonicalResponsesPayload = payload; -const serve = async (wantsStream: boolean) => await run( +/** How many times the chain asked the resolver for that payload. One stage owns the reading; + * anything below it reads the record. */ +let asked = 0; + +const serve = async (wantsStream: boolean) => await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream); + +const serveWith = async (gateway: ReturnType, wantsStream: boolean) => await run( responsesServePipeline(payload), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], @@ -109,11 +115,11 @@ const serve = async (wantsStream: boolean) => await run( 'serve.model': 'responses-model', }) as never, { - gateway: mockChatGatewayCtx({ wantsStream }), + gateway, background: () => {}, rememberCandidates: () => {}, rememberChatSelection: () => {}, - chatPayloadFor: () => affinityPayload, + chatPayloadFor: () => { asked += 1; return affinityPayload; }, selectAffinity: () => {}, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); @@ -131,6 +137,7 @@ const drain = async (rendered: unknown): Promise => { beforeEach(() => { affinityPayload = payload; + asked = 0; vi.mocked(enumerateModelCandidates).mockReset(); initRepo({ usage: { record: async () => {} }, @@ -141,25 +148,33 @@ beforeEach(() => { describe('the responses chain', () => { // Affinity rewrites client-carried state for the upstream that will see it, so the body // that goes out is the one it materialized rather than the one the client sent — and this - // chain dials the generate action, whatever else the protocol can ask an upstream for. - it('sends the payload affinity materialized for the candidate it dialled', async () => { + // chain dials the generate action, whatever else the protocol can ask an upstream for. The + // payload enters the record below the fork because everything between there and the dial + // rewrites the request as a fact, and one stage owns that reading: an ending that asked the + // resolver again would send a payload no stage in the chain had touched. + it('puts the payload this attempt is owed into the record, and sends that', async () => { let sent: Record | undefined; - let asked: unknown; + let action: unknown; affinityPayload = { ...payload, input: [{ type: 'message', role: 'user', content: 'rewritten' }], } as CanonicalResponsesPayload; - resolves([candidate(async (_model, body, action) => { + resolves([candidate(async (_model, body, asks) => { sent = body as Record; - asked = action; + action = asks; return stream(completed('hi')); })]); - await serve(false); + const { facts } = await serve(false); + // The exit contract names what the run answers with, and the request the chain sent is + // not that — but the record carries every key in flight, so what was sent is still there + // to read. + expect((facts as Record)['request.chat.responses']).toBe(affinityPayload); + expect(asked).toBe(1); expect(sent).toMatchObject({ input: [{ type: 'message', role: 'user', content: 'rewritten' }] }); expect(sent).not.toHaveProperty('model'); - expect(asked).toBe('generate'); + expect(action).toBe('generate'); }); // Every event goes out under its own SSE name, and the client's stream ends on the literal @@ -200,6 +215,49 @@ describe('the responses chain', () => { expect(billable).toEqual([expect.objectContaining({ quantities: { input_tokens: '11', output_tokens: '7' }, })]); + // A rate can depend on the tier and on how much input there was; both are selector + // coordinates, so they travel as pricing facts rather than as quantities. + expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 11 }); + }); + + // Time to first token is measured where the token is — a lifecycle envelope is not one, and + // a run that never stamps it is recorded as having produced nothing to time. + it('stamps the first frame that carried a generated token', async () => { + resolves([candidate(async () => stream(delta('hi'), completed('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + expect(gateway.attempt.firstOutputTokenAt).toBeNull(); + await drain(facts['response.chat.responses.rendered']); + + expect(gateway.attempt.firstOutputTokenAt).toBeTypeOf('number'); + }); + + // The turn is over at its terminal event, so what an upstream writes after it is not part + // of the answer and the client is not shown it. Reading stops there, which also closes the + // upstream rather than leaving the client's stream open behind a connection nobody reads. + it('stops reading at the terminal event', async () => { + resolves([candidate(async () => stream(delta('hi'), completed('hi'), delta(' and more')))]); + + const { facts } = await serve(true); + const frames = await drain(facts['response.chat.responses.rendered']); + + expect(frames.map(frame => frame.event)).toEqual([ + 'response.output_text.delta', + 'response.completed', + undefined, + ]); + }); + + // A stream that ran out before its terminal event is a turn nobody can answer from: the + // response was never stated complete, incomplete or failed, so serving what did arrive + // would report a truncated answer as a whole one. + it('fails a stream that ended without its terminal event', async () => { + resolves([candidate(async () => stream(delta('hi')))]); + + const { facts } = await serve(true); + + await expect(drain(facts['response.chat.responses.rendered'])).rejects.toThrow(RESPONSES_MISSING_TERMINAL_MESSAGE); }); // The upstream speaks SSE whatever the client asked for, so a client that did not ask to diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index ea71192914..90998d1dd4 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -4,6 +4,7 @@ // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate +// materializeAttempt puts the payload this candidate is owed into the record // callResponsesUpstream the ending: dials this candidate's wire // // One wire and one transport: `/v1/responses` over HTTP, dialled on a candidate's own @@ -20,8 +21,8 @@ // the same `response.chat.responses` key, which is what will make them interchangeable // with this one. // - this family's interceptors, the server-tool shims among them. Still only in the -// interceptor form, so the array between the fork and the ending is empty rather than -// short. +// interceptor form, so the array between the materialized payload and the ending is +// empty rather than short. // - the stored-items membrane: `previous_response_id` expansion, item hydration, and the // snapshot the next turn reads. It wraps the chain rather than sitting in it, and // `serve-prep.ts` and `client-output.ts` still hold it. @@ -45,8 +46,9 @@ import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts' import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; +import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; -import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import { doneFrame, renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { @@ -194,12 +196,16 @@ const callResponsesUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - // Affinity materializes the payload this candidate is owed: client-carried state — an - // encrypted reasoning blob, a compaction the upstream issued — is rewritten for the + // What the record holds by now: the payload affinity materialized for this candidate, as + // every stage between the fork and here has rewritten it. Client-carried state — an + // encrypted reasoning blob, a compaction the upstream issued — was rewritten for the // upstream that will see it, which is the whole reason a turn can be pinned at all. The // id the client addressed does not travel — the provider re-stamps whatever it resolved // upstream — and an alias' own rules apply to the body that is sent. - const asked = use.chatPayloadFor(facts['route.attempt']) as CanonicalResponsesPayload; + // + // The key holds what a client may send, whose `input` is a string or a list; this chain + // runs on the canonical form the entry normalized it to, which is the one a wire takes. + const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; const payload = { ...asked, model: candidate.model.id }; if (candidate.rules !== undefined) applyRulesToUpstreamResponses(payload, candidate.rules); const { model: _addressed, ...body } = payload; @@ -270,7 +276,7 @@ const callResponsesUpstream = defineStage< }); } - const metered = meterResponses(result.events, identity); + const metered = meterResponses(result.events, identity, use.gateway.attempt); return move({ ...facts, 'response.chat.responses': { kind: 'stream' as const, frames: metered.frames }, @@ -288,6 +294,7 @@ const callResponsesUpstream = defineStage< const meterResponses = ( source: AsyncIterable>, identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, ): { readonly frames: AsyncIterable>; readonly billable: Promise } => { let settle!: (billable: readonly BillableEntity[]) => void; const billable = new Promise(resolve => { settle = resolve; }); @@ -295,6 +302,11 @@ const meterResponses = ( let reported: BillableUsage | undefined; try { for await (const frame of source) { + // Time to first token is measured where the token is, which is the only place that + // knows a frame carries generated content rather than the envelope around it. + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'responses')) { + attempt.firstOutputTokenAt = performance.now(); + } if (frame.type !== 'event') { yield frame; continue; @@ -338,7 +350,6 @@ const billed = (identity: TelemetryModelIdentity, usage: BillableUsage | undefin * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ const narrowing = (payload: CanonicalResponsesPayload): ChatNarrowing> => ({ - source: 'responses', canServe: candidate => responsesTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeResponsesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /responses endpoint.`, @@ -382,5 +393,6 @@ export const responsesServePipeline = (payload: CanonicalResponsesPayload): Pipe failed: handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), owns: [], }), + materializeAttempt('request.chat.responses'), callResponsesUpstream, ]); From e24c91942ae4b993f2960dc4f15b02f43b5db744 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 03:28:51 +0800 Subject: [PATCH 021/156] fix(chat): translate, time, bill and end a Gemini turn as its chain states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ending read its turn back from the resolver while everything between the fork and the dial rewrites the request as a fact, so a rewrite would have gone nowhere. Affinity's per-candidate turn now enters the record below the fork, where there is one turn to speak of, and the ending translates the fact. Time to first token was written in one place no stage on this chain reaches, so every Gemini run took the neutral branch and recorded no TTFT at all. It is measured where the token is, on the dialect the upstream spoke — the same one the usage beside it is read on. A rate can depend on the service tier and on how much input there was. Both are selector coordinates rather than quantities, and dropping them left the pricer with an empty selector, so neither axis could ever apply. The reading now goes through the same measurement the other families use and carries both. A wire that closed cleanly had not thereby finished the turn: with no finish reason on any choice, nothing that came out of the translation said the answer was over, and a client streaming was served those frames as a whole answer. The folded shape already refused them, through the protocol's own reassembly; both shapes now do, because the ending stops at the turn's terminal frame and fails a stream that ran out before one. ChatNarrowing.source goes with the key it was declared under. --- .../data-plane/chat/gemini-pipeline_test.ts | 85 +++++++++++++++---- .../src/data-plane/chat/gemini/pipeline.ts | 83 +++++++++++------- 2 files changed, 122 insertions(+), 46 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index f0474efc1a..bf7a728b9b 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -12,7 +12,7 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload } from '@floway-dev/protocols/gemini'; +import { GEMINI_MISSING_TERMINAL_MESSAGE, type GeminiPayload } from '@floway-dev/protocols/gemini'; import { directFetcher, type ModelCandidate, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -80,6 +80,15 @@ const turn = (): AsyncIterable> => ({ })(), }); +/** The wire below closed cleanly and never finished the turn: no choice ever carried a + * finish reason, so nothing that comes out of the translation is a Gemini terminal event. */ +const unfinishedTurn = (): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + yield chunk({ content: 'he' }); + yield doneFrame(); + })(), +}); + const streamed = ( events: AsyncIterable>, headers?: Headers, @@ -93,6 +102,10 @@ const payload = { contents: [{ role: 'user' as const, parts: [{ text: 'hello' }] * turn the client sent, which is the point: carried state is rewritten per candidate. */ let affinityPayload: GeminiPayload = payload; +/** How many times the chain asked the resolver for that turn. One stage owns the reading; + * anything below it reads the record. */ +let asked = 0; + /** Which candidates the run said a follow-up turn must come back to. */ const selected: ModelCandidate[] = []; @@ -106,6 +119,7 @@ beforeEach(() => { recorded.performance = []; selected.length = 0; affinityPayload = payload; + asked = 0; initRepo({ usage: { record: async (row: unknown) => { recorded.usage.push(row); } }, performance: { @@ -115,15 +129,18 @@ beforeEach(() => { } as never); }); -const serve = async (facts: Record) => await run( +const serve = async (facts: Record) => + await serveWith(mockChatGatewayCtx({ wantsStream: facts['ingress.chat.gemini.wantsStream'] === true }), facts); + +const serveWith = async (gateway: ReturnType, facts: Record) => await run( geminiServePipeline(payload), move(facts) as never, { - gateway: mockChatGatewayCtx({ wantsStream: facts['ingress.chat.gemini.wantsStream'] === true }), + gateway, background: () => {}, rememberCandidates: () => {}, rememberChatSelection: () => {}, - chatPayloadFor: () => affinityPayload, + chatPayloadFor: () => { asked += 1; return affinityPayload; }, selectAffinity: (candidate: ModelCandidate) => { selected.push(candidate); }, resolveAttempt, } as never, @@ -145,11 +162,9 @@ const collect = async (rendered: unknown): Promise => { }; describe('the gemini pipeline', () => { - // The whole entry contract, and it does not mention `request.chat.gemini` — the stage that - // reads the turn is the ending, a stage whose only trait is `return` declares no request - // side at all, and so assembly cannot see what it reads. A caller who omits the payload - // gets a runtime failure at the deepest stage rather than the assembly error the entry - // contract exists to give it. + // The whole entry contract, and it does not mention `request.chat.gemini`: the turn the + // ending translates is not the one the caller handed in but the one materializeAttempt put + // into the record below the fork, which is where a per-candidate payload can exist at all. it('assembles, and asks its caller only for what the descending stages need', () => { expect([...geminiServePipeline(payload).entryNeeds].sort()).toEqual([ 'ingress.chat.gemini.wantsStream', @@ -184,8 +199,11 @@ describe('the gemini pipeline', () => { // Gemini carries its state in `thoughtSignature` parts, which only mean anything to the // upstream that issued them — so the turn that is translated is the one affinity rewrote - // for this candidate, and the candidate that answered is marked for the next turn. - it('translates what affinity materialized, and marks the candidate that answered', async () => { + // for this candidate, and the candidate that answered is marked for the next turn. That + // turn enters the record below the fork because everything between there and the dial + // rewrites the request as a fact, and one stage owns the reading: an ending that asked the + // resolver again would translate a turn no stage in the chain had touched. + it('translates what the record holds, and marks the candidate that answered', async () => { let sent: Record | undefined; affinityPayload = { contents: [{ role: 'user', parts: [{ text: 'rewritten' }] }] }; const dialled = candidate('up_a', async (_model, body) => { @@ -197,6 +215,10 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts()); await collect(facts['response.chat.gemini.rendered']); + // The exit contract names what the run answers with, and the turn the chain translated is + // not that — but the record carries every key in flight, so it is still there to read. + expect((facts as Record)['request.chat.gemini']).toBe(affinityPayload); + expect(asked).toBe(1); expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); expect(selected).toEqual([dialled]); await drain(); @@ -286,11 +308,40 @@ describe('the gemini pipeline', () => { expect(recorded.usage).toHaveLength(0); const usage = facts['response.chat.gemini.streamedUsage']; expect(usage).not.toBeNull(); - expect(await usage!).toEqual([ - { - identity: { model: 'gemini-2.5-pro', upstream: 'up_a', modelKey: 'gemini-2.5-pro-key', pricing: null }, - quantities: { input_tokens: '5', output_tokens: '7' }, - }, - ]); + const billable = await usage!; + expect(billable[0]).toMatchObject({ + identity: { model: 'gemini-2.5-pro', upstream: 'up_a', modelKey: 'gemini-2.5-pro-key', pricing: null }, + quantities: { input_tokens: '5', output_tokens: '7' }, + }); + // A rate can depend on the tier and on how much input there was; both are selector + // coordinates, so they travel as pricing facts rather than as quantities. + expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 5 }); + }); + + // Time to first token is measured where the token is, and on the dialect the upstream + // actually spoke rather than the one the client did. A run that never stamps it is recorded + // as having produced nothing to time. + it('stamps the first frame that carried a generated token', async () => { + resolves([candidate('up_a', async () => streamed(turn()))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts, drain } = await serveWith(gateway, entryFacts()); + expect(gateway.attempt.firstOutputTokenAt).toBeNull(); + await collect(facts['response.chat.gemini.rendered']); + + expect(gateway.attempt.firstOutputTokenAt).toBeTypeOf('number'); + await drain(); + }); + + // A wire that closes cleanly has not thereby finished the turn: with no finish reason on + // any choice, nothing that comes out of the translation says the answer is over. Serving + // those frames would report a truncated answer as a whole one. + it('fails a stream that ended without its terminal frame', async () => { + resolves([candidate('up_a', async () => streamed(unfinishedTurn()))]); + + const { facts, drain } = await serve(entryFacts()); + + await expect(collect(facts['response.chat.gemini.rendered'])).rejects.toThrow(GEMINI_MISSING_TERMINAL_MESSAGE); + await drain(); }); }); diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index e0f65aece7..5542f8d64c 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -4,6 +4,7 @@ // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate +// materializeAttempt puts the payload this candidate is owed into the record // callGeminiViaChatCompletions the ending: translates, dials, and meters what came back // // Gemini is source-only. Nothing translates *into* it, so it contributes one pipeline rather @@ -24,9 +25,10 @@ // this one: no stream, no billable turn, and a `{ totalTokens }` envelope of its own. // - the Gemini interceptors and the affinity egress. The four `interceptors/` entries and // the thought-signature rewrite on the way out are still only in the interceptor form. -// - a stream that ends without a terminal event. `collectGeminiProtocolEventsToResult` -// throws it, and here it leaves the run as a throw rather than as the 502 Google-RPC -// envelope `respond.ts` renders. +// - the Google-RPC envelope `respond.ts` wrote into a client's own stream for a turn that +// ended without a terminal event. The chain detects one — the ending stops reading at +// the terminal frame and throws when the frames run out before one — but it leaves the +// run as a throw rather than as that 502 envelope. // // One deliberate difference from `respond.ts`, shared with every other family on the // pipeline: an upstream that refused is answered in its own words. A Chat Completions wire @@ -36,21 +38,20 @@ // that envelope maps to `INTERNAL`. import { analyzeGeminiAffinity } from './affinity/ingress.ts'; -import type { UsageQuantities } from '../../../repo/types.ts'; -import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; +import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import { billableUsageFromChatCompletionsEvent } from '../chat-completions/usage.ts'; import type { ChatFacts } from '../facts.ts'; import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; +import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; -import { resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { renderGeminiError } from './errors.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; @@ -58,6 +59,8 @@ import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFr import { collectGeminiProtocolEventsToResult, geminiProtocolFrameToSSEFrame, + GEMINI_MISSING_TERMINAL_MESSAGE, + isGeminiTerminalEvent, type GeminiPayload, type GeminiStreamEvent, } from '@floway-dev/protocols/gemini'; @@ -176,7 +179,7 @@ const renderSSE = (frames: AsyncIterable>): Asy * that come out of it, so a figure the upstream stated is billed as stated. */ const callGeminiViaChatCompletions = defineStage< - C<'route.attempt' | 'ingress.http.headers'>, + C<'request.chat.gemini' | 'route.attempt' | 'ingress.http.headers'>, C<'response.chat.gemini' | 'response.chat.gemini.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, ChatServices @@ -198,14 +201,14 @@ const callGeminiViaChatCompletions = defineStage< use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); // The trip owns both directions: it builds the payload that goes out and hands back the - // closure that turns the upstream's frames into Gemini's. What it translates is what - // affinity materialized for this candidate — client-carried state rewritten for the - // upstream that will see it, which is the whole reason a turn can be pinned at all. The - // id the client addressed does not travel: Gemini carries it in the URL, the trip stamps - // the candidate's own, and the wire is handed the body without it. An alias' own rules - // apply to the translated body, which is the only shape the wire will see. - const asked = use.chatPayloadFor(facts['route.attempt']) as GeminiPayload; - const trip = await translateGeminiViaChatCompletions(asked, { model: candidate.model.id }); + // closure that turns the upstream's frames into Gemini's. What it translates is what the + // record holds by now — the turn affinity materialized for this candidate, as every stage + // between the fork and here has rewritten it, with client-carried state rewritten for the + // upstream that will see it. The id the client addressed does not travel: Gemini carries + // it in the URL, the trip stamps the candidate's own, and the wire is handed the body + // without it. An alias' own rules apply to the translated body, which is the only shape + // the wire will see. + const trip = await translateGeminiViaChatCompletions(facts['request.chat.gemini'], { model: candidate.model.id }); if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(trip.target, candidate.rules); const { model: _addressed, ...body } = trip.target; @@ -259,10 +262,10 @@ const callGeminiViaChatCompletions = defineStage< // This candidate answered, so it is the one a follow-up turn carrying our own state must // come back to. use.selectAffinity(candidate); - const metered = meterChatCompletions(result.events, identity); + const metered = meterChatCompletions(result.events, identity, use.gateway.attempt); return move({ ...facts, - 'response.chat.gemini': { kind: 'stream' as const, frames: trip.events(metered.frames) }, + 'response.chat.gemini': { kind: 'stream' as const, frames: geminiFramesUntilTerminal(trip.events(metered.frames)) }, 'response.chat.gemini.streamedUsage': metered.billable, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], @@ -276,6 +279,7 @@ const callGeminiViaChatCompletions = defineStage< const meterChatCompletions = ( source: AsyncIterable>, identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, ): { readonly frames: AsyncIterable>; readonly billable: Promise } => { let settle!: (billable: readonly BillableEntity[]) => void; const billable = new Promise(resolve => { settle = resolve; }); @@ -283,6 +287,12 @@ const meterChatCompletions = ( let reported: BillableUsage | undefined; try { for await (const frame of source) { + // Time to first token is measured where the token is, which is the only place that + // knows a frame carries generated content rather than the envelope around it — and + // it is read on the dialect the upstream spoke, like the usage beside it. + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'chat-completions')) { + attempt.firstOutputTokenAt = performance.now(); + } if (frame.type === 'event') { const usage = billableUsageFromChatCompletionsEvent(frame.event); if (usage !== null) reported = usage; @@ -293,29 +303,43 @@ const meterChatCompletions = ( // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle([{ identity, quantities: billed(reported) }]); + settle([billedEntity(reported, identity)]); } })(); return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; }; -/** An upstream that reported nothing leaves no quantities at all, which is a different - * statement from reporting zero. - * - * A billed entity is keyed by billing metric, which is not the shape a protocol reports in, - * so the reading is converted rather than cast. The service tier survives as far as - * `TokenUsage.tier` and no further: a billed entity is an identity and a bag of quantities, - * and the pricing selector the tier feeds has no seat there. */ -const billed = (usage: BillableUsage | undefined): UsageQuantities => { +/** Where a Gemini turn ends, stated on the frames the client is handed rather than on the + * wire below: a Chat Completions stream that closed cleanly without ever reporting a finish + * reason translates into candidates that never finish, and serving those would report a + * truncated answer as a whole one. */ +const geminiFramesUntilTerminal = (frames: AsyncIterable>): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + yield frame; + // The turn is over, so there is nothing further to read; returning here closes the + // translation and the wire under it. + if (frame.type === 'done' || isGeminiTerminalEvent(frame.event)) return; + } + throw new Error(GEMINI_MISSING_TERMINAL_MESSAGE); + })(), +}); + +/** What one attempt is billable for. An upstream that reported nothing leaves no quantities + * at all, which is a different statement from reporting zero — and a rate can depend on the + * service tier and on how much input there was, so both travel as pricing facts rather than + * being folded into the quantities. */ +const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { const tokens = tokenUsageFromBillableUsage(usage); - return tokens === null ? {} : tokenUsageQuantities(tokens); + if (tokens === null) return { identity, quantities: {} }; + const measurement = tokenUsageMeasurement(tokens); + return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; }; /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ const narrowing = (payload: GeminiPayload): ChatNarrowing> => ({ - source: 'gemini', canServe: candidate => geminiTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeGeminiAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the Gemini generateContent endpoint.`, @@ -345,5 +369,6 @@ export const geminiServePipeline = (payload: GeminiPayload): Pipeline isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini']), owns: [], }), + materializeAttempt('request.chat.gemini'), callGeminiViaChatCompletions, ]); From a98794d4dee017b12728811e15e0d01ab8bf899f Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 03:36:14 +0800 Subject: [PATCH 022/156] fix(chat): refuse a Chat Completions stream that never said it ended MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The meter read to exhaustion, so an upstream that dropped mid-turn was served as a whole answer and anything it sent after its terminator was forwarded. The streaming path is where this has to be caught: nothing folds those frames, so the collector's own check never sees them. The terminator is written out before the read stops, because it is what a client reads as the end. The first test written for this passed with the guard deleted — it drove the non-streaming path, where the collector throws the same sentence for its own reasons. It now drains the stream, and both halves of the guard are covered. --- .../chat/chatcompletions-pipeline_test.ts | 37 ++++++++++++++++++- .../chat/chat-completions/pipeline.ts | 15 +++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index f12ab16ef1..87968683de 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -10,7 +10,7 @@ import { enumerateModelCandidates } from '../../../src/data-plane/providers/reso import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import { CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, type ChatCompletionsPayload, type ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; import type { SseFrame } from '@floway-dev/protocols/common'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -56,6 +56,16 @@ const chunk = (text: string): ChatCompletionsStreamEvent => ({ choices: [{ index: 0, delta: { content: text }, finish_reason: null }], }); +/** A stream that stops without ever saying it ended — what a dropped upstream looks like. */ +const truncated = (...events: readonly ChatCompletionsStreamEvent[]): ProviderStreamResult => ({ + ok: true, + modelKey: 'chat-model-key', + headers: new Headers(), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event }; + })(), +}); + const stream = (...events: readonly ChatCompletionsStreamEvent[]): ProviderStreamResult => ({ ok: true, modelKey: 'chat-model-key', @@ -243,6 +253,31 @@ describe('the chat completions chain', () => { }); }); + // Serving what arrived would present a truncated answer as a whole one, and a client has no + // way to tell the difference. The streaming path is where this has to be caught: nothing + // folds those frames, so the collector's own check never runs on them. + it('fails a stream that ran out without saying it ended', async () => { + resolves([candidate(async () => truncated(chunk('he'), chunk('llo')))]); + + const { facts } = await serve(true); + const drain = async (): Promise => { + for await (const _frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { /* to the end */ } + }; + + await expect(drain()).rejects.toThrow(CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE); + }); + + // Anything an upstream keeps sending after its terminator is not part of the answer. + it('stops reading at the terminator, and writes it', async () => { + resolves([candidate(async () => stream(chunk('hi')))]); + + const { facts } = await serve(true); + const frames: SseFrame[] = []; + for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); + + expect(frames.at(-1)!.data).toBe('[DONE]'); + }); + // A refused connection is an outcome the fork has to be able to see, not a fault that ends // the run — so the second candidate is tried and its answer is the one served. it('fails a dial that never connected over to the next candidate', async () => { diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 423fed40aa..e16292cfea 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -34,6 +34,8 @@ import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import { + CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, + chatCompletionsErrorPayloadMessage, chatCompletionsProtocolFrameToSSEFrame, collectChatCompletionsProtocolEventsToResult, type ChatCompletionsPayload, @@ -259,8 +261,6 @@ const meterChatCompletions = ( let reported: BillableUsage | undefined; try { for await (const frame of source) { - // Time to first token is measured where the token is, which is the only place that - // knows a frame carries generated content rather than the envelope around it. // Time to first token is measured where the token is, which is the only place that // knows a frame carries generated content rather than the envelope around it. if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'chat-completions')) { @@ -271,7 +271,13 @@ const meterChatCompletions = ( if (usage !== null) reported = usage; } yield frame; + // The terminator is written out before the read stops, because it is what the client + // reads as the end. Stopping here also drops anything an upstream sends after it. + if (isTerminal(frame)) return; } + // A stream that ran out without saying it ended is not a turn that finished. Serving + // what arrived would present a truncated answer as a whole one. + throw new Error(CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE); } finally { // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are @@ -282,6 +288,11 @@ const meterChatCompletions = ( return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; }; +/** The frame that says this turn is over: the transport's own terminator, or an error the + * upstream wrote into the stream instead of finishing it. */ +const isTerminal = (frame: ProtocolFrame): boolean => + frame.type === 'done' || (frame.type === 'event' && chatCompletionsErrorPayloadMessage(frame.event) !== null); + /** What one attempt is billable for. An upstream that reported nothing leaves no quantities * at all, which is a different statement from reporting zero — and a rate can depend on the * service tier and on how much input there was, so both travel as pricing facts rather than From b3b4e1139312bb8eaeddba351b78156cf3842f76 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 06:09:35 +0800 Subject: [PATCH 023/156] feat(chat): serve /v1/chat/completions through its pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chain existed and was tested against stub providers, but nothing reached it: the route still built a chat context and ran the onion, so every property the chain states was stated only under test. The handler is now the prologue and epilogue the other migrated families use — read the ingress, open the chat prologue against a scratchpad store, hand the payload over, and write what the run answered with. Two decisions stay at the entry because only it can make them, and both are read before any stage can rewrite what they are read from: whether the client asked to stream, which the run has to be opened knowing, and whether it asked to be shown the usage chunk metering asks the upstream for either way. A body that is not JSON never enters a pipeline at all — there is no model to resolve and no attempt to make — so it is answered in this protocol's own 400 envelope. Wiring is what states what the chain does not carry yet, and two of them are written down in failing tests rather than papered over. The affinity egress never runs, so a turn hands back no `reasoning_opaque` and the next one cannot be pinned to the upstream that served this one; `http_test.ts` says so twice and is left saying it. Six of the nine interceptors are not stages — the usage-chunk request, both usage normalizers and the three vendor normalizers — a refusal that reached no upstream leaves the streamed-usage key unwritten, and the meter reports what was billed without reporting whether the stream reached its terminator. Each of those is the chain's own to close. --- .../data-plane/chat/chat-completions/http.ts | 129 ++++++++++-------- 1 file changed, 74 insertions(+), 55 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/chat-completions/http.ts b/packages/gateway/src/data-plane/chat/chat-completions/http.ts index 43ccb6b999..149a3f5326 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/http.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/http.ts @@ -1,66 +1,85 @@ -import { translatorInputErrorResult } from './errors.ts'; -import { respondChatCompletions } from './respond.ts'; -import { chatCompletionsServe } from './serve.ts'; +// POST /v1/chat/completions, served through the pipeline. +// +// The handler is a prologue and an epilogue around `chatCompletionsServePipeline`: read what +// the client sent, hand it over, and turn what the run answered with into a response. +// Everything between is stages. +// +// Two things are decided here because only the entry knows them, and both are read before any +// stage can rewrite the payload they are read from: whether the client asked to stream, which +// the run has to be opened knowing, and whether it asked to be shown the usage chunk that +// metering asks the upstream for on every streaming turn. + +import { chatCompletionsServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; -import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; +import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; +import { openChatPrologue } from '../prologue.ts'; import { createNonResponsesSourceStore } from '../responses/items/store.ts'; -import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerModelsUnavailableResponse } from '../shared/upstream-models-error.ts'; +import { move } from '@floway-dev/pipeline'; import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; -import { TranslatorInputError } from '@floway-dev/translate'; - -// Surfaces a pre-stream throw (malformed JSON body, an interceptor crash, -// etc.) as a Chat Completions-shaped 502 with the same internal-error -// envelope the in-flow `internal-error` ExecuteResult produces. A -// `ProviderModelsUnavailableError` carrying an upstream HTTP body relays -// that body verbatim — the upstream's `/models` 401 IS the diagnostic. The -// caller passes its outer `ctx` when one was already constructed (so the -// dump row preserves the model attribution the request-time -// `requestedModel` stamped, and the throwing-candidate telemetry stamped -// in serve.ts survives onto the error row); a fresh ctx is minted only -// for pre-parse failures where no payload was available to read model from. -const respondWithInternalError = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - const verbatim = providerModelsUnavailableResponse(error); - if (verbatim !== null) return verbatim; - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const result = internalErrorResult(502, toInternalDebugError(error), effectiveCtx.attempt.telemetry); - const response = await respondChatCompletions(c, result, false, false, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); -}; -// Pre-stream caller-input failure raised by a translator → Chat -// Completions-shaped 400 invalid_request_error envelope. Anything else -// falls through to the internal-error 502 path. -const respondToThrow = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - if (!(error instanceof TranslatorInputError)) return await respondWithInternalError(c, error, requestBody, ctx); - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const response = await respondChatCompletions(c, translatorInputErrorResult(error, effectiveCtx.attempt.telemetry), false, false, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); +/** A body the gateway could not read is reported in the words the protocol's own clients + * parse, rather than as a fault of the gateway's. */ +const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: ChatCompletionsPayload } | { type: 'invalid'; message: string } => { + try { + return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as ChatCompletionsPayload }; + } catch (error) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; + } }; export const chatCompletionsHttp = { generate: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = JSON.parse(new TextDecoder().decode(requestBody.bytes)) as ChatCompletionsPayload; - const wantsStream = payload.stream === true; - // Read the caller's intent BEFORE any interceptor mutates - // `payload.stream_options.include_usage`. Capturing it here means the - // downstream renderer never needs to consult per-request Hono context - // slots — the value lives in this http-entry closure for the duration of - // the request. - const includeUsageChunk = payload.stream_options?.include_usage === true; - ctx = createChatGatewayCtxFromHono(c, { wantsStream, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonResponsesSourceStore(apiKey.id)); - const result = await chatCompletionsServe.generate({ payload, ctx, headers: inboundHeaders(c) }); - const response = await respondChatCompletions(c, result, wantsStream, includeUsageChunk, ctx); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') { + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + refused.gateway, + Response.json({ error: { message: request.message, type: 'invalid_request_error' } }, { status: 400 }), + ); } + + const { payload } = request; + const wantsStream = payload.stream === true; + const prologue = openChatPrologue(c, ingress, { + wantsStream, + model: payload.model, + storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + chatCompletionsServePipeline(payload), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'chatCompletions', + 'ingress.chat.chatCompletions.wantsStream': wantsStream, + 'ingress.chat.chatCompletions.wantsUsageChunk': payload.stream_options?.include_usage === true, + 'request.chat.chatCompletions': payload, + 'serve.model': payload.model, + }) as never, + facts => { + const rendered = facts['response.chat.chatCompletions.rendered']; + if (isFrames(rendered)) return { frames: rendered }; + return { body: JSON.stringify(rendered), contentType: 'application/json' }; + }, + facts => { + const streamed = facts['response.chat.chatCompletions.streamedUsage']; + // Two readings this chain does not hand up. A refusal that never reached an upstream + // leaves the key unwritten rather than null, and the meter that does write it resolves + // what was billed without saying whether the frames reached their terminator — so a + // stream that stopped short settles here as one that finished. Both belong to the + // chain: a family whose meter reports the outcome hands up a `StreamOutcome`, and this + // adapter goes with it. + return streamed === null || streamed === undefined + ? null + : streamed.then(billable => ({ billable, failed: false })); + }, + ); }, }; From 8d0b11a1f6a4202fedaa8f5140142a45191af3ee Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 06:09:47 +0800 Subject: [PATCH 024/156] feat(chat): serve /v1/messages through its pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chain existed and was tested against stub providers, but nothing reached it. The generate entry is now the prologue and epilogue the other migrated families use; `/v1/messages/count_tokens` stays where it is, because it is a second operation over this protocol rather than another wire under its chain. A stream is handed to the seam with this protocol's own idle frame. Anthropic defines a `ping` event and its clients read one, so an idle connection is held open with that rather than with an SSE comment no client sees — the chain names the frame and the route passes it on. A body that is not JSON is answered in Anthropic's own envelope, and a body carrying `anthropic_beta` or `betas` is refused as before, both now recorded as the gateway refusals they are. Wiring states what the chain does not carry yet. None of this family's five interceptors is a stage, and the first of them is the one `http_test.ts` catches: Claude Code's one-token model probe is answered by dialling an upstream instead of by the gateway. The web-search shim, the billing-attribution scrub, the reasoning and role rewrites, and the affinity egress are absent with it; so is the mid-stream `error` event `respond.ts` wrote into a client's own stream. A refusal that reached no upstream leaves the streamed-usage key unwritten, and the meter reports what was billed without reporting whether the stream reached its terminator. --- .../src/data-plane/chat/messages/http.ts | 93 ++++++++++++++++--- 1 file changed, 78 insertions(+), 15 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/messages/http.ts b/packages/gateway/src/data-plane/chat/messages/http.ts index 626a7cc805..8ac5838849 100644 --- a/packages/gateway/src/data-plane/chat/messages/http.ts +++ b/packages/gateway/src/data-plane/chat/messages/http.ts @@ -1,14 +1,26 @@ -import { translatorInputErrorResult } from './errors.ts'; +// POST /v1/messages, served through the pipeline, and POST /v1/messages/count_tokens beside +// it — a second operation over this protocol rather than another wire under its chain, so it +// still goes through `messagesServe.countTokens`. +// +// The generate entry is a prologue and an epilogue around `messagesServePipeline`: read what +// the client sent, hand it over, and turn what the run answered with into a response. +// Everything between is stages. + +import { renderMessagesError, translatorInputErrorResult } from './errors.ts'; +import { messagesKeepAlive, messagesServePipeline } from './pipeline.ts'; import { respondMessages } from './respond.ts'; import { messagesServe } from './serve.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; +import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../../shared/gateway-ctx.ts'; import { inboundHeaders } from '../../shared/inbound-headers.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; +import { openChatPrologue } from '../prologue.ts'; import { createNonResponsesSourceStore } from '../responses/items/store.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; import { providerModelsUnavailableResponse } from '../shared/upstream-models-error.ts'; +import { move } from '@floway-dev/pipeline'; import type { MessagesPayload } from '@floway-dev/protocols/messages'; import { internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; import { TranslatorInputError } from '@floway-dev/translate'; @@ -62,26 +74,77 @@ const respondToThrow = async (c: AuthedContext, error: unknown, requestBody: Req return finalizeGatewayResponse(effectiveCtx, response); }; +/** A body the gateway could not read is reported in the words this protocol's own clients + * parse, rather than as a fault of the gateway's. */ +const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: MessagesPayload } | { type: 'invalid'; message: string } => { + try { + return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as MessagesPayload }; + } catch (error) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; + } +}; + +/** A request the gateway refused before it reached a pipeline: there is no model to resolve + * and no attempt to make, so there is nothing for a run to record beyond the refusal. */ +const refuse = (c: AuthedContext, ingress: Ingress, response: Response): Response => { + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse(refused.gateway, response); +}; + const parsePayload = (requestBody: RequestBody): MessagesPayload => JSON.parse(new TextDecoder().decode(requestBody.bytes)) as MessagesPayload; export const messagesHttp = { generate: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = parsePayload(requestBody); - const rejected = rejectBodyBetaResponse(payload); - if (rejected) return rejected; - - const wantsStream = payload.stream === true; - ctx = createChatGatewayCtxFromHono(c, { wantsStream, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonResponsesSourceStore(apiKey.id)); - const result = await messagesServe.generate({ payload, ctx, headers: inboundHeaders(c) }); - const response = await respondMessages(c, result, wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') { + return refuse(c, ingress, Response.json(renderMessagesError(400, request.message), { status: 400 })); } + + const { payload } = request; + const rejected = rejectBodyBetaResponse(payload); + if (rejected !== null) return refuse(c, ingress, rejected); + + const wantsStream = payload.stream === true; + const prologue = openChatPrologue(c, ingress, { + wantsStream, + model: payload.model, + storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + messagesServePipeline(payload), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'messages', + 'ingress.chat.messages.wantsStream': wantsStream, + 'request.chat.messages': payload, + 'serve.model': payload.model, + }) as never, + facts => { + const rendered = facts['response.chat.messages.rendered']; + // Anthropic defines a `ping` event and its clients read one, so an idle connection is + // held open with that rather than with a comment no client sees. + if (isFrames(rendered)) return { frames: rendered, keepAlive: messagesKeepAlive }; + return { body: JSON.stringify(rendered), contentType: 'application/json' }; + }, + facts => { + const streamed = facts['response.chat.messages.streamedUsage']; + // Two readings this chain does not hand up. A refusal that never reached an upstream + // leaves the key unwritten rather than null, and the meter that does write it resolves + // what was billed without saying whether the frames reached their terminator — so a + // stream that stopped short settles here as one that finished. Both belong to the + // chain: a family whose meter reports the outcome hands up a `StreamOutcome`, and this + // adapter goes with it. + return streamed === null || streamed === undefined + ? null + : streamed.then(billable => ({ billable, failed: false })); + }, + ); }, countTokens: async (c: AuthedContext): Promise => { From d3e8739c6c795c4d7e284e3a6060e97e64a37800 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 06:09:57 +0800 Subject: [PATCH 025/156] feat(chat): serve :generateContent through its pipeline The chain existed and was tested against stub providers, but nothing reached it. Both generate actions are now the prologue and epilogue the other migrated families use; `:countTokens` stays where it is, because it is a second operation over this protocol rather than another wire under its chain. Gemini carries the model in the path rather than the body, so the id the run resolves against is the one the route already split off the action segment, and whether the turn streams is the action itself rather than a field. A body that is not JSON is answered in the Google-RPC envelope this protocol's clients read. Wiring states what the chain does not carry yet, and `http_test.ts` catches the first: the chain has one wire, so a candidate reachable only over Messages or Responses is dialled on Chat Completions anyway and the turn fails. None of the four Gemini interceptors is a stage, the thought-signature rewrite and the affinity egress on the way out are absent with them, and a stream that ends without a terminal event ends the run as a throw rather than as the Google-RPC envelope `respond.ts` wrote into the client's own stream. A refusal that reached no upstream leaves the streamed-usage key unwritten, and the meter reports what was billed without reporting whether the stream reached its terminator. --- .../src/data-plane/chat/gemini/http.ts | 84 ++++++++++++++++--- 1 file changed, 73 insertions(+), 11 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/gemini/http.ts b/packages/gateway/src/data-plane/chat/gemini/http.ts index d9e0b03efd..fd14f22a8f 100644 --- a/packages/gateway/src/data-plane/chat/gemini/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini/http.ts @@ -1,13 +1,26 @@ -import { translatorInputErrorResult } from './errors.ts'; +// `/v1beta/models/:modelAction`, whose `:generateContent` and `:streamGenerateContent` +// actions are served through the pipeline. `:countTokens` is a second operation over this +// protocol rather than another wire under its chain, so it still goes through +// `geminiServe.countTokens`. +// +// The generate entry is a prologue and an epilogue around `geminiServePipeline`: read what +// the client sent, hand it over, and turn what the run answered with into a response. +// Everything between is stages. + +import { renderGeminiError, translatorInputErrorResult } from './errors.ts'; +import { geminiServePipeline } from './pipeline.ts'; import { geminiInternalRpcErrorResponse, geminiRpcErrorResponse, respondGemini } from './respond.ts'; import { geminiServe } from './serve.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; +import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { inboundHeaders } from '../../shared/inbound-headers.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; +import { openChatPrologue } from '../prologue.ts'; import { createNonResponsesSourceStore } from '../responses/items/store.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; +import { move } from '@floway-dev/pipeline'; import type { GeminiContent, GeminiPayload } from '@floway-dev/protocols/gemini'; import { internalErrorResult, ProviderModelsUnavailableError, toInternalDebugError } from '@floway-dev/provider'; import { TranslatorInputError } from '@floway-dev/translate'; @@ -99,21 +112,70 @@ export const geminiHttp = async (c: AuthedContext): Promise => { return geminiRpcErrorResponse(404, `Unknown Gemini model action: ${parsed.action}`); }; -const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: boolean): Promise => { - const requestBody = await readRequestBody(c); - const payload = parseGeminiBodyBytes(requestBody, body => body as GeminiPayload); - if (payload instanceof Response) return payload; - - const ctx = createChatGatewayCtxFromHono(c, { wantsStream, requestBody: takeRequestBody(requestBody), model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonResponsesSourceStore(apiKey.id)); +/** A body the gateway could not read is reported in the words this protocol's own clients + * parse, rather than as a fault of the gateway's. */ +const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: GeminiPayload } | { type: 'invalid'; message: string } => { try { - const result = await geminiServe.generate({ payload, ctx, model, headers: inboundHeaders(c) }); - const response = await respondGemini(c, result, wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); + return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as GeminiPayload }; } catch (error) { - return await respondWithGeminiError(c, error, ctx, wantsStream); + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } }; +const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: boolean): Promise => { + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') { + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + refused.gateway, + Response.json(renderGeminiError(400, request.message), { status: 400 }), + ); + } + + const { payload } = request; + const prologue = openChatPrologue(c, ingress, { + wantsStream, + model, + storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + geminiServePipeline(payload), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'gemini', + 'ingress.chat.gemini.wantsStream': wantsStream, + 'request.chat.gemini': payload, + // Gemini carries the model in the path rather than the body, so the id the run + // resolves against is the one the route split off. + 'serve.model': model, + }) as never, + facts => { + const rendered = facts['response.chat.gemini.rendered']; + if (isFrames(rendered)) return { frames: rendered }; + return { body: JSON.stringify(rendered), contentType: 'application/json' }; + }, + facts => { + const streamed = facts['response.chat.gemini.streamedUsage']; + // Two readings this chain does not hand up. A refusal that never reached an upstream + // leaves the key unwritten rather than null, and the meter that does write it resolves + // what was billed without saying whether the frames reached their terminator — so a + // stream that stopped short settles here as one that finished. Both belong to the + // chain: a family whose meter reports the outcome hands up a `StreamOutcome`, and this + // adapter goes with it. + return streamed === null || streamed === undefined + ? null + : streamed.then(billable => ({ billable, failed: false })); + }, + ); +}; + const runGeminiCountTokens = async (c: AuthedContext, model: string): Promise => { const requestBody = await readRequestBody(c); const payload = parseGeminiBodyBytes(requestBody, parseGeminiCountTokensPayload); From eb42b17c7e1c7bd19810ec856784bfc193ee1831 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 06:23:58 +0800 Subject: [PATCH 026/156] fix(chat): settle a chat turn by how its stream ended, and let a refusal settle at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects the non-chat families had already been through, both found by wiring the routes. Every chat chain handed its usage up as a bare list, so each handler adapted it with failed: false — a stream that stopped before its terminator was recorded as a turn that produced what it said it would. The meters now report both together, which is what the seam has taken since the non-chat families moved, and the four adapters delete. A refusal never provided the streamed-usage key at all, so settlement's still-reading test read undefined and deferred a turn that had no stream to wait for: the run wrote no performance row. Refusing now says there is nothing still to read, which is the same statement an ending makes when it did not stream. --- .../chat/chatcompletions-pipeline_test.ts | 8 +-- .../data-plane/chat/gemini-pipeline_test.ts | 3 +- .../data-plane/chat/messages-pipeline_test.ts | 3 +- .../chat/responses-pipeline_test.ts | 3 +- .../data-plane/chat/chat-completions/http.ts | 13 +---- .../chat/chat-completions/pipeline.ts | 25 +++++---- .../src/data-plane/chat/gemini/http.ts | 13 +---- .../src/data-plane/chat/gemini/pipeline.ts | 52 +++++++++++++------ .../src/data-plane/chat/messages/http.ts | 13 +---- .../src/data-plane/chat/messages/pipeline.ts | 33 +++++++----- .../src/data-plane/chat/responses/pipeline.ts | 25 +++++---- 11 files changed, 101 insertions(+), 90 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index 87968683de..4ef9e535d2 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -218,12 +218,14 @@ describe('the chat completions chain', () => { const { facts } = await serve(true); for await (const _frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { /* drain */ } - const billable = await facts['response.chat.chatCompletions.streamedUsage']!; + const outcome = await facts['response.chat.chatCompletions.streamedUsage']!; - expect(billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); + expect(outcome.billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); // A rate can depend on the tier and on how much input there was; both are selector // coordinates, so they travel as pricing facts rather than as quantities. - expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 11 }); + expect(outcome.billable[0]!.pricingFacts).toMatchObject({ inputTokens: 11 }); + // The stream reached its terminator, so the turn produced what it said it would. + expect(outcome.failed).toBe(false); }); // Time to first token is measured where the token is — an envelope frame is not one, and a diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index bf7a728b9b..8286c21917 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -308,7 +308,8 @@ describe('the gemini pipeline', () => { expect(recorded.usage).toHaveLength(0); const usage = facts['response.chat.gemini.streamedUsage']; expect(usage).not.toBeNull(); - const billable = await usage!; + const outcome = await usage!; + const billable = outcome.billable; expect(billable[0]).toMatchObject({ identity: { model: 'gemini-2.5-pro', upstream: 'up_a', modelKey: 'gemini-2.5-pro-key', pricing: null }, quantities: { input_tokens: '5', output_tokens: '7' }, diff --git a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts index 0b55dfd2f4..1b874177b3 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts @@ -290,7 +290,8 @@ describe('the messages chain', () => { ]); await collect(facts['response.chat.messages.rendered']); - const billable = await facts['response.chat.messages.streamedUsage']!; + const outcome = await facts['response.chat.messages.streamedUsage']!; + const billable = outcome.billable; expect(billable[0]).toMatchObject({ identity: { model: 'claude-model', upstream: 'up_a', modelKey: 'claude-model-key', pricing: null }, diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index c2bfbf5efc..ab32a169a4 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -210,7 +210,8 @@ describe('the responses chain', () => { const { facts } = await serve(true); await drain(facts['response.chat.responses.rendered']); - const billable = await facts['response.chat.responses.streamedUsage']!; + const outcome = await facts['response.chat.responses.streamedUsage']!; + const billable = outcome.billable; expect(billable).toEqual([expect.objectContaining({ quantities: { input_tokens: '11', output_tokens: '7' }, diff --git a/packages/gateway/src/data-plane/chat/chat-completions/http.ts b/packages/gateway/src/data-plane/chat/chat-completions/http.ts index 149a3f5326..9d965bb2c7 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/http.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/http.ts @@ -68,18 +68,7 @@ export const chatCompletionsHttp = { if (isFrames(rendered)) return { frames: rendered }; return { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - facts => { - const streamed = facts['response.chat.chatCompletions.streamedUsage']; - // Two readings this chain does not hand up. A refusal that never reached an upstream - // leaves the key unwritten rather than null, and the meter that does write it resolves - // what was billed without saying whether the frames reached their terminator — so a - // stream that stopped short settles here as one that finished. Both belong to the - // chain: a family whose meter reports the outcome hands up a `StreamOutcome`, and this - // adapter goes with it. - return streamed === null || streamed === undefined - ? null - : streamed.then(billable => ({ billable, failed: false })); - }, + facts => facts['response.chat.chatCompletions.streamedUsage'], ); }, }; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index e16292cfea..54fa336a71 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -16,6 +16,7 @@ import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; import { billableUsageFromChatCompletionsEvent } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; @@ -55,7 +56,7 @@ export interface ChatCompletionsFacts extends ChatFacts { 'response.chat.chatCompletions.rendered': Record | AsyncIterable; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ - 'response.chat.chatCompletions.streamedUsage': Promise | null; + 'response.chat.chatCompletions.streamedUsage': Promise | null; } type C = { [P in K]: ChatCompletionsFacts[P] }; @@ -240,7 +241,7 @@ const callChatCompletionsUpstream = defineStage< return move({ ...facts, 'response.chat.chatCompletions': { kind: 'stream' as const, frames: metered.frames }, - 'response.chat.chatCompletions.streamedUsage': metered.billable, + 'response.chat.chatCompletions.streamedUsage': metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); @@ -254,9 +255,12 @@ const meterChatCompletions = ( source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, -): { readonly frames: AsyncIterable>; readonly billable: Promise } => { - let settle!: (billable: readonly BillableEntity[]) => void; - const billable = new Promise(resolve => { settle = resolve; }); +): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { + let settle!: (outcome: StreamOutcome) => void; + const outcome = new Promise(resolve => { settle = resolve; }); + // Running out without the terminal frame is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; const generator = (async function* () { let reported: BillableUsage | undefined; try { @@ -273,7 +277,7 @@ const meterChatCompletions = ( yield frame; // The terminator is written out before the read stops, because it is what the client // reads as the end. Stopping here also drops anything an upstream sends after it. - if (isTerminal(frame)) return; + if (isTerminal(frame)) { sawTerminal = true; return; } } // A stream that ran out without saying it ended is not a turn that finished. Serving // what arrived would present a truncated answer as a whole one. @@ -282,10 +286,10 @@ const meterChatCompletions = ( // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle([billedEntity(reported, identity)]); + settle({ billable: [billedEntity(reported, identity)], failed: !sawTerminal }); } })(); - return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; + return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; }; /** The frame that says this turn is over: the transport's own terminator, or an error the @@ -307,11 +311,12 @@ const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModel /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: ChatCompletionsPayload): ChatNarrowing> => ({ +const narrowing = (payload: ChatCompletionsPayload): ChatNarrowing> => ({ canServe: candidate => chatCompletionsTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeChatCompletionsAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /chat/completions endpoint.`, refuse: (status, message, reason) => ({ + 'response.chat.chatCompletions.streamedUsage': null, 'response.chat.chatCompletions': { status, message, @@ -326,7 +331,7 @@ const narrowing = (payload: ChatCompletionsPayload): ChatNarrowing { - const streamed = facts['response.chat.gemini.streamedUsage']; - // Two readings this chain does not hand up. A refusal that never reached an upstream - // leaves the key unwritten rather than null, and the meter that does write it resolves - // what was billed without saying whether the frames reached their terminator — so a - // stream that stopped short settles here as one that finished. Both belong to the - // chain: a family whose meter reports the outcome hands up a `StreamOutcome`, and this - // adapter goes with it. - return streamed === null || streamed === undefined - ? null - : streamed.then(billable => ({ billable, failed: false })); - }, + facts => facts['response.chat.gemini.streamedUsage'], ); }; diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 5542f8d64c..44c2f7830e 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -38,8 +38,10 @@ // that envelope maps to `INTERNAL`. import { analyzeGeminiAffinity } from './affinity/ingress.ts'; +import { renderGeminiError } from './errors.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; @@ -52,7 +54,6 @@ import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { renderGeminiError } from './errors.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; @@ -78,7 +79,7 @@ export interface GeminiFacts extends ChatFacts { 'response.chat.gemini.rendered': Record | AsyncIterable; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ - 'response.chat.gemini.streamedUsage': Promise | null; + 'response.chat.gemini.streamedUsage': Promise | null; } type C = { [P in K]: GeminiFacts[P] }; @@ -262,11 +263,14 @@ const callGeminiViaChatCompletions = defineStage< // This candidate answered, so it is the one a follow-up turn carrying our own state must // come back to. use.selectAffinity(candidate); - const metered = meterChatCompletions(result.events, identity, use.gateway.attempt); + // Where this turn ends is stated on the Gemini frames the client is handed, and what it + // billed is read on the Chat Completions wire below them. + const ending = { sawTerminal: false }; + const metered = meterChatCompletions(result.events, identity, use.gateway.attempt, ending); return move({ ...facts, - 'response.chat.gemini': { kind: 'stream' as const, frames: geminiFramesUntilTerminal(trip.events(metered.frames)) }, - 'response.chat.gemini.streamedUsage': metered.billable, + 'response.chat.gemini': { kind: 'stream' as const, frames: geminiFramesUntilTerminal(trip.events(metered.frames), ending) }, + 'response.chat.gemini.streamedUsage': metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); @@ -280,9 +284,13 @@ const meterChatCompletions = ( source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, -): { readonly frames: AsyncIterable>; readonly billable: Promise } => { - let settle!: (billable: readonly BillableEntity[]) => void; - const billable = new Promise(resolve => { settle = resolve; }); + ending: { sawTerminal: boolean }, +): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { + let settle!: (outcome: StreamOutcome) => void; + const outcome = new Promise(resolve => { settle = resolve; }); + // Running out without the terminal frame is what "it did not finish" means, and it is known + // at the same moment the usage is. + const sawTerminal = false; const generator = (async function* () { let reported: BillableUsage | undefined; try { @@ -303,23 +311,30 @@ const meterChatCompletions = ( // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle([billedEntity(reported, identity)]); + settle({ billable: [billedEntity(reported, identity)], failed: !ending.sawTerminal }); } })(); - return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; + return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; }; /** Where a Gemini turn ends, stated on the frames the client is handed rather than on the * wire below: a Chat Completions stream that closed cleanly without ever reporting a finish * reason translates into candidates that never finish, and serving those would report a * truncated answer as a whole one. */ -const geminiFramesUntilTerminal = (frames: AsyncIterable>): AsyncIterable> => ({ +const geminiFramesUntilTerminal = ( + frames: AsyncIterable>, + ending: { sawTerminal: boolean }, +): AsyncIterable> => ({ [Symbol.asyncIterator]: () => (async function* () { for await (const frame of frames) { yield frame; // The turn is over, so there is nothing further to read; returning here closes the - // translation and the wire under it. - if (frame.type === 'done' || isGeminiTerminalEvent(frame.event)) return; + // translation and the wire under it. The end is stated here and the usage is read on + // the wire below, so one flag carries what the settlement needs from both. + if (frame.type === 'done' || isGeminiTerminalEvent(frame.event)) { + ending.sawTerminal = true; + return; + } } throw new Error(GEMINI_MISSING_TERMINAL_MESSAGE); })(), @@ -339,12 +354,17 @@ const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModel /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: GeminiPayload): ChatNarrowing> => ({ +const narrowing = (payload: GeminiPayload): ChatNarrowing> => ({ canServe: candidate => geminiTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeGeminiAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the Gemini generateContent endpoint.`, - refuse: (status, message) => ({ 'response.chat.gemini': { status, message } }), - refuses: ['response.chat.gemini'], + refuse: (status, message) => ({ + 'response.chat.gemini': { status, message }, + // A refusal never opened a stream, so there is nothing still to read — which is what + // lets settlement write its row here rather than wait for numbers that never come. + 'response.chat.gemini.streamedUsage': null, + }), + refuses: ['response.chat.gemini', 'response.chat.gemini.streamedUsage'], }); export type GeminiServeEntry = C< diff --git a/packages/gateway/src/data-plane/chat/messages/http.ts b/packages/gateway/src/data-plane/chat/messages/http.ts index 8ac5838849..ce1691781b 100644 --- a/packages/gateway/src/data-plane/chat/messages/http.ts +++ b/packages/gateway/src/data-plane/chat/messages/http.ts @@ -132,18 +132,7 @@ export const messagesHttp = { if (isFrames(rendered)) return { frames: rendered, keepAlive: messagesKeepAlive }; return { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - facts => { - const streamed = facts['response.chat.messages.streamedUsage']; - // Two readings this chain does not hand up. A refusal that never reached an upstream - // leaves the key unwritten rather than null, and the meter that does write it resolves - // what was billed without saying whether the frames reached their terminator — so a - // stream that stopped short settles here as one that finished. Both belong to the - // chain: a family whose meter reports the outcome hands up a `StreamOutcome`, and this - // adapter goes with it. - return streamed === null || streamed === undefined - ? null - : streamed.then(billable => ({ billable, failed: false })); - }, + facts => facts['response.chat.messages.streamedUsage'], ); }, diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index c5703d8373..ec0795263a 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -15,9 +15,11 @@ // the array between the materialized payload and the ending is empty rather than short. import { analyzeMessagesAffinity } from './affinity/ingress.ts'; +import { renderMessagesError } from './errors.ts'; import { createMessagesBillableUsageReader } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; @@ -29,7 +31,6 @@ import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { renderMessagesError } from './errors.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import { renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { @@ -53,7 +54,7 @@ export interface MessagesFacts extends ChatFacts { 'response.chat.messages.rendered': Record | AsyncIterable; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ - 'response.chat.messages.streamedUsage': Promise | null; + 'response.chat.messages.streamedUsage': Promise | null; } type M = { [P in K]: MessagesFacts[P] }; @@ -238,7 +239,7 @@ const callMessagesUpstream = defineStage< return move({ ...facts, 'response.chat.messages': { kind: 'stream' as const, frames: metered.frames }, - 'response.chat.messages.streamedUsage': metered.billable, + 'response.chat.messages.streamedUsage': metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); @@ -253,9 +254,12 @@ const meterMessages = ( source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, -): { readonly frames: AsyncIterable>; readonly billable: Promise } => { - let settle!: (billable: readonly BillableEntity[]) => void; - const billable = new Promise(resolve => { settle = resolve; }); +): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { + let settle!: (outcome: StreamOutcome) => void; + const outcome = new Promise(resolve => { settle = resolve; }); + // Running out without the terminal frame is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; const readBillableUsage = createMessagesBillableUsageReader(); const generator = (async function* () { let reported: BillableUsage | undefined; @@ -274,7 +278,7 @@ const meterMessages = ( // The turn is over, so there is nothing further to read. An upstream that holds the // connection open past `message_stop` would otherwise hold the client's stream open // with it; returning here closes the read, which cancels the upstream. - if (isMessagesTerminalFrame(frame)) return; + if (isMessagesTerminalFrame(frame)) { sawTerminal = true; return; } } // Frames ran out with no terminal event, which is a turn nobody can answer from: the // message was never stopped and never failed. @@ -283,10 +287,10 @@ const meterMessages = ( // Reached however the frames ended — the terminal event, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle([billedEntity(reported, identity)]); + settle({ billable: [billedEntity(reported, identity)], failed: !sawTerminal }); } })(); - return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; + return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; }; /** What ends a Messages turn. Anthropic's own stream terminator is an event rather than a @@ -309,12 +313,17 @@ const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModel /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: MessagesPayload): ChatNarrowing> => ({ +const narrowing = (payload: MessagesPayload): ChatNarrowing> => ({ canServe: candidate => messagesTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeMessagesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /messages endpoint.`, - refuse: (status, message) => ({ 'response.chat.messages': { status, message } }), - refuses: ['response.chat.messages'], + refuse: (status, message) => ({ + 'response.chat.messages': { status, message }, + // A refusal never opened a stream, so there is nothing still to read — which is what + // lets settlement write its row here rather than wait for numbers that never come. + 'response.chat.messages.streamedUsage': null, + }), + refuses: ['response.chat.messages', 'response.chat.messages.streamedUsage'], }); /** What this protocol writes on an idle connection. Anthropic defines a `ping` event and diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index 90998d1dd4..01b1bbe07b 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -38,6 +38,7 @@ import { analyzeResponsesAffinity } from './affinity/ingress.ts'; import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; @@ -72,7 +73,7 @@ export interface ResponsesFacts extends ChatFacts { 'response.chat.responses.rendered': Record | AsyncIterable; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ - 'response.chat.responses.streamedUsage': Promise | null; + 'response.chat.responses.streamedUsage': Promise | null; } type R = { [P in K]: ResponsesFacts[P] }; @@ -280,7 +281,7 @@ const callResponsesUpstream = defineStage< return move({ ...facts, 'response.chat.responses': { kind: 'stream' as const, frames: metered.frames }, - 'response.chat.responses.streamedUsage': metered.billable, + 'response.chat.responses.streamedUsage': metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); @@ -295,9 +296,12 @@ const meterResponses = ( source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, -): { readonly frames: AsyncIterable>; readonly billable: Promise } => { - let settle!: (billable: readonly BillableEntity[]) => void; - const billable = new Promise(resolve => { settle = resolve; }); +): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { + let settle!: (outcome: StreamOutcome) => void; + const outcome = new Promise(resolve => { settle = resolve; }); + // Running out without the terminal frame is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; const generator = (async function* () { let reported: BillableUsage | undefined; try { @@ -317,7 +321,7 @@ const meterResponses = ( // The turn is over, so there is nothing further to read. An upstream that holds the // connection open past its terminal event would otherwise hold the client's stream // open with it; returning here closes the read, which cancels the upstream. - if (isResponsesTerminalEvent(frame.event)) return; + if (isResponsesTerminalEvent(frame.event)) { sawTerminal = true; return; } } // Frames ran out with no terminal event, which is a turn nobody can answer from: the // response was never stated complete, incomplete or failed. @@ -326,10 +330,10 @@ const meterResponses = ( // Reached however the frames ended — the terminal event, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle([billed(identity, reported)]); + settle({ billable: [billed(identity, reported)], failed: !sawTerminal }); } })(); - return { frames: { [Symbol.asyncIterator]: () => generator }, billable }; + return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; }; /** An upstream that reported nothing leaves no quantities at all, which is a different @@ -349,11 +353,12 @@ const billed = (identity: TelemetryModelIdentity, usage: BillableUsage | undefin /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: CanonicalResponsesPayload): ChatNarrowing> => ({ +const narrowing = (payload: CanonicalResponsesPayload): ChatNarrowing> => ({ canServe: candidate => responsesTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeResponsesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /responses endpoint.`, refuse: (status, message, reason) => ({ + 'response.chat.responses.streamedUsage': null, 'response.chat.responses': { status, message, @@ -368,7 +373,7 @@ const narrowing = (payload: CanonicalResponsesPayload): ChatNarrowing Date: Mon, 17 Aug 2026 06:25:46 +0800 Subject: [PATCH 027/156] feat(chat): run the Messages, Gemini and Responses interceptors as stages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chat Completions had three of its interceptors as stages; the other three families had none, so their chains ran nothing between the fork and the ending. Gemini's thought suppression was among the missing, which is a client that never opted in being shown thought parts. Thirteen stages, one per (interceptor, protocol) pair because the payload types differ, and one rule each: the role fold, the flag reading and the key removal are written once, and a protocol contributes only the walk over its own items. Two of them speak about the response — Gemini's thought suppression and the Responses cache-token fold — and say so as a response-direction declaration, carrying what they read on the way down in the stage's own closure, because a response-side `needs` can only name what the ending provides. What the interceptor form did by mutation is a rewrite now: the record is frozen, so Gemini's three strippers cannot delete a field in place. Every rule writes conditionally, so a turn it does not touch comes back by identity and the layer costs what it actually changed. A `ctx.targetApi` guard is gone wherever array position now says the same thing. Four interceptors stay in the interceptor form. The Claude Code probe answers with the Messages family's own response facts rather than rewriting a payload, and the Responses compact shim, the Responses server-tool shim and the Messages web-search shim each drive a turn of their own. Composing the stages into each family's chain is the next step and is not here. --- .../data-plane/chat-interceptors_test.ts | 500 ++++++++++- .../src/data-plane/chat/interceptors.ts | 817 +++++++++++++++++- 2 files changed, 1239 insertions(+), 78 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index a4a31f62a4..7b4619b446 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -1,49 +1,96 @@ -// The three interceptors three chat protocols share, migrated. Two properties matter and -// neither is about the rewrites themselves — those are transcribed from code that already -// worked. What is new is that a stage returns a record instead of assigning to a context, -// and that a rewrite which changes nothing changes nothing by identity. +// The chat interceptors, migrated. Two properties matter and neither is about the rewrites +// themselves — those are transcribed from code that already worked. What is new is that a +// stage returns a record instead of assigning to a context, and that a rewrite which changes +// nothing changes nothing by identity. A rule that speaks about the response direction has a +// third: what it read on the way down has to reach the way back, which is the only thing its +// closure is for. import { describe, expect, it } from 'vitest'; -import type { Chat } from '../../src/data-plane/chat/facts.ts'; import { applyRoleCompatibilityToChatCompletions, + applyRoleCompatibilityToMessages, + applyRoleCompatibilityToResponses, disableReasoningOnForcedToolChoiceForChatCompletions, + disableReasoningOnForcedToolChoiceForMessages, + disableReasoningOnForcedToolChoiceForResponses, + normalizeExclusiveCachedTokensForResponses, + stripBillingAttributionFromMessages, stripPromptCacheKeyForChatCompletions, + stripPromptCacheKeyForResponses, + stripSafetySettingsFromGemini, + stripUnsupportedPartFieldsFromGemini, + stripUnsupportedToolsFromGemini, + suppressThoughtPartsFromGemini, + vendorDeepSeekNormalizeForResponses, + vendorQwenNormalizeForResponses, } from '../../src/data-plane/chat/interceptors.ts'; import type { AttemptSelector } from '../../src/data-plane/pipeline/facts.ts'; -import { move, run, compose, defineStage } from '@floway-dev/pipeline'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; +import type { MessagesPayload } from '@floway-dev/protocols/messages'; +import type { ResponsesInputItem, ResponsesPayload, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; /** What an attempt is, to a stage that only reads flags. The selector carries them as data, * so nothing here needs the live candidate the resolver kept. */ const attemptWith = (...flags: string[]): AttemptSelector => ({ upstreamId: 'u', modelId: 'm', flags }); -const payload = (messages: ChatCompletionsPayload['messages'], rest: Partial = {}): ChatCompletionsPayload => - ({ model: 'm', messages, ...rest }) as ChatCompletionsPayload; - -/** Runs one stage and hands back what it passed down, which is the only thing these - * stages do — none of them touches the response direction. The sink has to provide the - * answer the stage above declared it needs, or `compose` refuses the array; that refusal - * is the assembly check doing its job and is why the harness looks like this. */ -const handedDown = async (stage: unknown, facts: object): Promise> => { - let seen: Record = {}; - const sink = defineStage>({ +/** + * Runs one stage and hands back both directions: the record it passed down, and the record + * that came back up. + * + * The sink has to answer at the key the stage above declared it needs, or `compose` refuses + * the array — that refusal is the assembly check doing its job, and is why the harness is + * told which key a protocol answers at rather than guessing one. + */ +const runStage = async ( + stage: unknown, + responseKey: string, + facts: Record, + answer: unknown = { kind: 'value' as const, body: null }, +): Promise<{ readonly down: Record; readonly up: Record }> => { + let down: Record = {}; + const sink = defineStage, Record>({ name: 'sink', - return: { provides: ['response.chat.chatCompletions'] }, + return: { provides: [responseKey] }, execute: async received => { - seen = received as Record; - return move({ ...received, 'response.chat.chatCompletions': { kind: 'value' as const, body: null } }); + down = received; + return move({ ...received, [responseKey]: answer }); }, }); - await run(compose('one', [stage as never, sink]), move(facts) as never, {}); - return seen; + const { facts: up } = await run(compose('one', [stage as never, sink]), move(facts) as never, {}); + return { down, up: up as Record }; +}; + +/** An answer in the shape a wire hands up: frames, read once. */ +const streamOf = (frames: readonly ProtocolFrame[]) => ({ + kind: 'stream' as const, + frames: { [Symbol.asyncIterator]: () => (async function* () { yield* frames; })() }, +}); + +const drain = async (answer: unknown): Promise[]> => { + const collected: ProtocolFrame[] = []; + for await (const frame of (answer as { frames: AsyncIterable> }).frames) collected.push(frame); + return collected; }; -describe('the shared chat interceptors, as stages', () => { +const chatCompletionsPayload = ( + messages: ChatCompletionsPayload['messages'], + rest: Partial = {}, +): ChatCompletionsPayload => ({ model: 'm', messages, ...rest }) as ChatCompletionsPayload; + +const messagesPayload = (messages: unknown[], rest: Record = {}): MessagesPayload => + ({ model: 'm', max_tokens: 16, messages, ...rest }) as unknown as MessagesPayload; + +const responsesPayload = (input: unknown[], rest: Record = {}): ResponsesPayload => + ({ model: 'm', input, ...rest }) as unknown as ResponsesPayload; + +describe('the Chat Completions interceptors, as stages', () => { it('rewrites the roles its flags name, in the settled order', async () => { - const down = await handedDown(applyRoleCompatibilityToChatCompletions, { - 'request.chat.chatCompletions': payload([ + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([ { role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }, { role: 'system', content: 'mid' }, @@ -51,12 +98,12 @@ describe('the shared chat interceptors, as stages', () => { 'route.attempt': attemptWith('rewrite-system-to-developer'), }); const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; - expect(after.messages.map(m => m.role)).toEqual(['developer', 'user', 'developer']); + expect(after.messages.map(message => message.role)).toEqual(['developer', 'user', 'developer']); }); it('turns only a mid-conversation system message into a user one', async () => { - const down = await handedDown(applyRoleCompatibilityToChatCompletions, { - 'request.chat.chatCompletions': payload([ + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([ { role: 'system', content: 'lead' }, { role: 'system', content: 'still lead' }, { role: 'user', content: 'hi' }, @@ -65,15 +112,15 @@ describe('the shared chat interceptors, as stages', () => { 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), }); const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; - expect(after.messages.map(m => m.role)).toEqual(['system', 'system', 'user', 'user']); + expect(after.messages.map(message => message.role)).toEqual(['system', 'system', 'user', 'user']); }); // The convention the whole exact-recording story rests on. A stage whose flags do not // fire must hand the payload back as the *same object*, or every layer costs a copy of // the conversation and the dump stops being affordable. it('hands the same payload back by identity when no flag fires', async () => { - const original = payload([{ role: 'system', content: 'lead' }]); - const down = await handedDown(applyRoleCompatibilityToChatCompletions, { + const original = chatCompletionsPayload([{ role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { 'request.chat.chatCompletions': original, 'route.attempt': attemptWith(), }); @@ -81,8 +128,8 @@ describe('the shared chat interceptors, as stages', () => { }); it('hands the same payload back when a flag fires but changes nothing', async () => { - const original = payload([{ role: 'user', content: 'hi' }]); - const down = await handedDown(applyRoleCompatibilityToChatCompletions, { + const original = chatCompletionsPayload([{ role: 'user', content: 'hi' }]); + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { 'request.chat.chatCompletions': original, 'route.attempt': attemptWith('rewrite-system-to-developer'), }); @@ -90,11 +137,11 @@ describe('the shared chat interceptors, as stages', () => { }); it('shares every message it did not rewrite', async () => { - const messages = Array.from({ length: 49 }, (_, i) => ( - i === 0 ? { role: 'system' as const, content: 'lead' } : { role: 'user' as const, content: `turn ${i}` } + const messages = Array.from({ length: 49 }, (_, index) => ( + index === 0 ? { role: 'system' as const, content: 'lead' } : { role: 'user' as const, content: `turn ${index}` } )); - const original = payload(messages); - const down = await handedDown(applyRoleCompatibilityToChatCompletions, { + const original = chatCompletionsPayload(messages); + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { 'request.chat.chatCompletions': original, 'route.attempt': attemptWith('rewrite-system-to-developer'), }); @@ -104,14 +151,14 @@ describe('the shared chat interceptors, as stages', () => { }); it('disables reasoning only when a tool choice is forced', async () => { - const forced = await handedDown(disableReasoningOnForcedToolChoiceForChatCompletions, { - 'request.chat.chatCompletions': payload([], { tool_choice: 'required' }), + const { down: forced } = await runStage(disableReasoningOnForcedToolChoiceForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([], { tool_choice: 'required' }), 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); expect((forced['request.chat.chatCompletions'] as ChatCompletionsPayload).reasoning_effort).toBe('none'); - const free = payload([], { tool_choice: 'auto' }); - const untouched = await handedDown(disableReasoningOnForcedToolChoiceForChatCompletions, { + const free = chatCompletionsPayload([], { tool_choice: 'auto' }); + const { down: untouched } = await runStage(disableReasoningOnForcedToolChoiceForChatCompletions, 'response.chat.chatCompletions', { 'request.chat.chatCompletions': free, 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); @@ -119,8 +166,8 @@ describe('the shared chat interceptors, as stages', () => { }); it('removes the prompt cache key, and the key itself rather than its value', async () => { - const down = await handedDown(stripPromptCacheKeyForChatCompletions, { - 'request.chat.chatCompletions': payload([], { prompt_cache_key: 'k' } as Partial), + const { down } = await runStage(stripPromptCacheKeyForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([], { prompt_cache_key: 'k' } as Partial), 'route.attempt': attemptWith('strip-prompt-cache-key'), }); const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; @@ -129,11 +176,376 @@ describe('the shared chat interceptors, as stages', () => { }); it('leaves a payload that never had the key alone, by identity', async () => { - const original = payload([]); - const down = await handedDown(stripPromptCacheKeyForChatCompletions, { + const original = chatCompletionsPayload([]); + const { down } = await runStage(stripPromptCacheKeyForChatCompletions, 'response.chat.chatCompletions', { 'request.chat.chatCompletions': original, 'route.attempt': attemptWith('strip-prompt-cache-key'), }); expect(down['request.chat.chatCompletions']).toBe(original); }); }); + +describe('the Messages interceptors, as stages', () => { + it('turns every inline system message into a user one', async () => { + const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.messages', { + 'request.chat.messages': messagesPayload([ + { role: 'system', content: 'lead' }, + { role: 'user', content: 'hi' }, + { role: 'system', content: 'mid' }, + ]), + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), + }); + const after = down['request.chat.messages'] as MessagesPayload; + // The top-level `system` field is the only first-position slot on this protocol, so even + // the leading one is past the leading run. + expect(after.messages.map(message => message.role)).toEqual(['user', 'user', 'user']); + }); + + it('hands the same conversation back by identity when the flag does not fire', async () => { + const original = messagesPayload([{ role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.messages', { + 'request.chat.messages': original, + 'route.attempt': attemptWith(), + }); + expect(down['request.chat.messages']).toBe(original); + }); + + it('hands the same conversation back when the flag fires but there is no system message', async () => { + const original = messagesPayload([{ role: 'user', content: 'hi' }]); + const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.messages', { + 'request.chat.messages': original, + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), + }); + expect(down['request.chat.messages']).toBe(original); + }); + + it('disables thinking on a forced tool choice and leaves the structured-output format', async () => { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.messages', { + 'request.chat.messages': messagesPayload([], { + tool_choice: { type: 'any' }, + output_config: { effort: 'high', format: { type: 'json_schema', schema: {} } }, + }), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + const after = down['request.chat.messages'] as MessagesPayload; + expect(after.thinking).toEqual({ type: 'disabled' }); + // Forced tool choice composes fine with structured output; it is thinking it does not + // compose with, so only the effort goes. + expect(after.output_config).toEqual({ format: { type: 'json_schema', schema: {} } }); + }); + + it('drops an output_config that held nothing but the effort', async () => { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.messages', { + 'request.chat.messages': messagesPayload([], { + tool_choice: { type: 'tool', name: 'f' }, + output_config: { effort: 'high' }, + }), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + expect('output_config' in (down['request.chat.messages'] as MessagesPayload)).toBe(false); + }); + + it('leaves a free tool choice alone, by identity', async () => { + const free = messagesPayload([], { tool_choice: { type: 'auto' } }); + const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.messages', { + 'request.chat.messages': free, + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + expect(down['request.chat.messages']).toBe(free); + }); + + it('scrubs the billing header line and the per-turn hash out of the system prompt', async () => { + const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.messages', { + 'request.chat.messages': messagesPayload([], { + system: 'x-anthropic-billing-header: cch=abcdef12\nreal instructions\ncch=beef01;', + }), + 'route.attempt': attemptWith('strip-billing-attribution'), + }); + expect((down['request.chat.messages'] as MessagesPayload).system).toBe('real instructions'); + }); + + it('drops a system block the scrub emptied, and the field when nothing is left', async () => { + const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.messages', { + 'request.chat.messages': messagesPayload([], { + system: [{ type: 'text', text: 'x-anthropic-billing-header: cch=abcdef12' }], + }), + 'route.attempt': attemptWith('strip-billing-attribution'), + }); + // An empty system prompt is not a system prompt: the field goes rather than riding on + // empty. + expect('system' in (down['request.chat.messages'] as MessagesPayload)).toBe(false); + }); + + it('leaves a system prompt carrying no attribution block alone, by identity', async () => { + const original = messagesPayload([], { system: [{ type: 'text', text: 'real instructions' }] }); + const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.messages', { + 'request.chat.messages': original, + 'route.attempt': attemptWith('strip-billing-attribution'), + }); + expect(down['request.chat.messages']).toBe(original); + }); +}); + +describe('the Gemini interceptors, as stages', () => { + it('drops the part fields no translation carries, and a part they were all of', async () => { + const { down } = await runStage(stripUnsupportedPartFieldsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': { + contents: [{ + role: 'user', + parts: [ + { text: 'hi', fileData: { mimeType: 'text/plain', fileUri: 'u' } }, + { fileData: { mimeType: 'text/plain', fileUri: 'u' } }, + ], + }], + systemInstruction: { parts: [{ text: 'sys', codeExecutionResult: {} }] }, + } satisfies GeminiPayload, + }); + const after = down['request.chat.gemini'] as GeminiPayload; + expect(after.contents?.[0]?.parts).toEqual([{ text: 'hi' }]); + expect(after.systemInstruction?.parts).toEqual([{ text: 'sys' }]); + }); + + it('hands a payload carrying no unsupported part field back by identity', async () => { + const original: GeminiPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; + const { down } = await runStage(stripUnsupportedPartFieldsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': original, + }); + expect(down['request.chat.gemini']).toBe(original); + }); + + it('keeps a tool group only for its function declarations', async () => { + const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': { + tools: [{ functionDeclarations: [{ name: 'f' }], googleSearch: {} }, { codeExecution: {} }], + } satisfies GeminiPayload, + }); + expect((down['request.chat.gemini'] as GeminiPayload).tools).toEqual([{ functionDeclarations: [{ name: 'f' }] }]); + }); + + it('removes the tool list itself when no group declared a function', async () => { + const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': { tools: [{ googleSearch: {} }] } satisfies GeminiPayload, + }); + // An empty tool list is a different request from no tools at all. + expect('tools' in (down['request.chat.gemini'] as GeminiPayload)).toBe(false); + }); + + it('hands a payload whose groups only declare functions back by identity', async () => { + const original: GeminiPayload = { tools: [{ functionDeclarations: [{ name: 'f' }] }] }; + const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': original, + }); + expect(down['request.chat.gemini']).toBe(original); + }); + + it('removes the safety settings, and leaves a payload that carries none alone', async () => { + const { down } = await runStage(stripSafetySettingsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': { safetySettings: [{ category: 'c', threshold: 't' }] } satisfies GeminiPayload, + }); + expect('safetySettings' in (down['request.chat.gemini'] as GeminiPayload)).toBe(false); + + const original: GeminiPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; + const { down: untouched } = await runStage(stripSafetySettingsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': original, + }); + expect(untouched['request.chat.gemini']).toBe(original); + }); + + it('hides thought parts from a caller who did not opt in', async () => { + const answer = streamOf([ + eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }] } }] }), + eventFrame({ + candidates: [{ + index: 0, + content: { parts: [{ text: 'more pondering', thought: true }, { text: 'the answer' }] }, + finishReason: 'STOP', + }], + }), + ]); + const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': {} satisfies GeminiPayload, + }, answer); + const frames = await drain(up['response.chat.gemini']); + // The first event was pure thought and had not finished, so it is nothing at all and + // never reaches the client; the second keeps only what the caller asked for. + expect(frames).toHaveLength(1); + const event = (frames[0] as { event: Extract }).event; + expect(event.candidates?.[0]?.content.parts).toEqual([{ text: 'the answer' }]); + }); + + // The opt-in is something the client sent, so it is read on the way down and used on the + // way back — the closure carrying it across is the only mechanism this proves. + it('leaves the stream alone when the caller asked to see thoughts', async () => { + const answer = streamOf([ + eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }] }, finishReason: 'STOP' }] }), + ]); + const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': { generationConfig: { thinkingConfig: { includeThoughts: true } } } satisfies GeminiPayload, + }, answer); + expect(up['response.chat.gemini']).toBe(answer); + }); + + it('leaves an answer that is not a stream alone, by identity', async () => { + const answer = { kind: 'value' as const, body: null }; + const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.gemini', { + 'request.chat.gemini': {} satisfies GeminiPayload, + }, answer); + expect(up['response.chat.gemini']).toBe(answer); + }); +}); + +describe('the Responses interceptors, as stages', () => { + const rolesOf = (down: Record): unknown[] => + ((down['request.chat.responses'] as ResponsesPayload).input as ResponsesInputItem[]) + .map(item => (item as { role?: unknown }).role); + + it('rewrites the roles its flags name, in the settled order', async () => { + const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.responses', { + 'request.chat.responses': responsesPayload([ + { type: 'message', role: 'system', content: 'lead' }, + { type: 'message', role: 'user', content: 'hi' }, + { type: 'message', role: 'system', content: 'mid' }, + ]), + 'route.attempt': attemptWith('rewrite-system-to-developer'), + }); + expect(rolesOf(down)).toEqual(['developer', 'user', 'developer']); + }); + + // The ordering the rule depends on: an item with no role of its own still ends the leading + // system run, which is what makes the system message after it mid-conversation. + it('lets an item carrying no role cross the leading system run', async () => { + const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.responses', { + 'request.chat.responses': responsesPayload([ + { type: 'message', role: 'system', content: 'lead' }, + { type: 'reasoning', id: 'rs_1', summary: [] }, + { type: 'message', role: 'system', content: 'after' }, + ]), + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), + }); + expect(rolesOf(down)).toEqual(['system', undefined, 'user']); + }); + + it('hands the same payload back by identity when no flag fires', async () => { + const original = responsesPayload([{ type: 'message', role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.responses', { + 'request.chat.responses': original, + 'route.attempt': attemptWith(), + }); + expect(down['request.chat.responses']).toBe(original); + }); + + it('disables reasoning on a required tool choice and on a named one', async () => { + for (const toolChoice of ['required', { type: 'function', name: 'f' }]) { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForResponses, 'response.chat.responses', { + 'request.chat.responses': responsesPayload([], { tool_choice: toolChoice, reasoning: { effort: 'high', summary: 'auto' } }), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + // The object is replaced rather than merged: a summary has nothing to summarize once + // reasoning is off. + expect((down['request.chat.responses'] as ResponsesPayload).reasoning).toEqual({ effort: 'none' }); + } + }); + + it('leaves an automatic tool choice alone, by identity', async () => { + const free = responsesPayload([], { tool_choice: 'auto' }); + const { down } = await runStage(disableReasoningOnForcedToolChoiceForResponses, 'response.chat.responses', { + 'request.chat.responses': free, + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + expect(down['request.chat.responses']).toBe(free); + }); + + it('removes the prompt cache key, and leaves a payload that never had it alone', async () => { + const { down } = await runStage(stripPromptCacheKeyForResponses, 'response.chat.responses', { + 'request.chat.responses': responsesPayload([], { prompt_cache_key: 'k' }), + 'route.attempt': attemptWith('strip-prompt-cache-key'), + }); + expect('prompt_cache_key' in (down['request.chat.responses'] as ResponsesPayload)).toBe(false); + + const original = responsesPayload([]); + const { down: untouched } = await runStage(stripPromptCacheKeyForResponses, 'response.chat.responses', { + 'request.chat.responses': original, + 'route.attempt': attemptWith('strip-prompt-cache-key'), + }); + expect(untouched['request.chat.responses']).toBe(original); + }); + + it('folds the cache buckets into the input total when the totals witness it', async () => { + // 100 + 50 + 10 = 160 reaches the stated total and 100 + 10 does not, so the upstream + // reports the buckets outside the input count whatever anyone flagged. + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); + const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.responses', { + 'route.attempt': attemptWith(), + }, answer); + const frames = await drain(up['response.chat.responses']); + expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 150, output_tokens: 10 }); + }); + + // The flag settles the responses whose totals witness nothing, which makes it a declaration + // input rather than the gate — and it arrives on the attempt this stage read on the way + // down, so this is also what proves the closure carries it to the way back. + it('folds on the flag alone when the response states no total', async () => { + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, input_tokens_details: { cached_tokens: 50, cache_write_tokens: 5 } })]); + const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.responses', { + 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), + }, answer); + const frames = await drain(up['response.chat.responses']); + expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 155 }); + }); + + it('hands a frame back by identity when the totals say the buckets are already inside', async () => { + const frame = usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 110, input_tokens_details: { cached_tokens: 50 } }); + const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.responses', { + 'route.attempt': attemptWith(), + }, streamOf([frame])); + const frames = await drain(up['response.chat.responses']); + expect(frames[0]).toBe(frame); + }); + + it('leaves an answer that is not a stream alone, by identity', async () => { + const answer = { kind: 'value' as const, body: null }; + const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.responses', { + 'route.attempt': attemptWith(), + }, answer); + expect(up['response.chat.responses']).toBe(answer); + }); + + it('puts the reasoning sentinel into the shape DeepSeek reads', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForResponses, 'response.chat.responses', { + 'request.chat.responses': responsesPayload([], { reasoning: { effort: 'none' } }), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + const after = down['request.chat.responses'] as Record; + expect('reasoning' in after).toBe(false); + expect(after.thinking).toEqual({ type: 'disabled' }); + }); + + it('puts the reasoning sentinel into the shape Qwen reads', async () => { + const { down } = await runStage(vendorQwenNormalizeForResponses, 'response.chat.responses', { + 'request.chat.responses': responsesPayload([], { reasoning: { effort: 'none' } }), + 'route.attempt': attemptWith('vendor-qwen'), + }); + const after = down['request.chat.responses'] as Record; + expect('reasoning' in after).toBe(false); + expect(after.enable_thinking).toBe(false); + }); + + it('leaves a payload that asked for real reasoning alone, by identity', async () => { + const original = responsesPayload([], { reasoning: { effort: 'high' } }); + for (const stage of [vendorDeepSeekNormalizeForResponses, vendorQwenNormalizeForResponses]) { + const { down } = await runStage(stage, 'response.chat.responses', { + 'request.chat.responses': original, + 'route.attempt': attemptWith('vendor-deepseek', 'vendor-qwen'), + }); + expect(down['request.chat.responses']).toBe(original); + } + }); +}); + +/** A lifecycle envelope carrying nothing but the counts under test. Every event that carries + * a response resource repeats the whole resource, which is why the rewrite is per event. */ +const usageEvent = (usage: Record): ProtocolFrame => + eventFrame({ type: 'response.completed', sequence_number: 0, response: { usage } } as unknown as ResponsesStreamEvent); + +const usageOf = (frame: ProtocolFrame): Record => + (frame as { event: { response: { usage: Record } } }).event.response.usage; diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index a72fb51f84..ad6e7b920a 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -1,25 +1,60 @@ -// The interceptors three of the four chat protocols share, as stages. Each is written -// against one protocol's payload key and dropped into that protocol's array — one stage -// per (interceptor, protocol) pair, because the payload types differ, but one *rule* each, -// written once here. +// The chat interceptors, as stages. Each is written against one protocol's payload key and +// dropped into that protocol's array — one stage per (interceptor, protocol) pair, because +// the payload types differ, but one *rule* each, written once here: the role fold, the flag +// reading and the key removal are shared, and only the walk over a protocol's own items is +// per protocol. // -// What changes from the interceptor form is the two things the pipeline replaced. A stage +// What changes from the interceptor form is the three things the pipeline replaced. A stage // returns the record it hands on instead of assigning to `ctx.payload`, so a rewrite is a -// spread and structural sharing is what makes it cheap — today's code already writes the -// map conditionally, which is the convention that keeps a 49-message conversation costing -// three objects. And `ctx.targetApi` is a declared need rather than an ambient field — array -// position says which chain a stage is in, and what it reads about the protocol it is -// looking at is in the record. The guards themselves are not deleted by that: there is one -// interceptor array, so a rule that must not run on a re-entered request still says so. +// spread and structural sharing is what makes it cheap — every rule here writes +// conditionally, which is the convention that keeps a 49-message conversation costing three +// objects, and it is also what lets a Gemini payload be rewritten at all now that the record +// is frozen and `delete ctx.payload.x` would throw. `ctx.targetApi` is a declared need rather +// than an ambient field — array position says which chain a stage is in, and what it reads +// about the protocol it is looking at is in the record. The guards themselves are not deleted +// by that: there is one interceptor array, so a rule that must not run on a re-entered request +// still says so. And a rule that reads the response — the two that rewrite a stream — states +// it as a response-direction declaration rather than as work after `await run()`, with what it +// learned on the way down carried in the stage's own closure, because a response-side `needs` +// can only name what the ending provides. import type { Chat } from './facts.ts'; +import { asJsonObject, type JsonObject, readJsonNumber } from '../../shared/json-helpers.ts'; +import { isFailure, type AttemptSelector } from '../pipeline/facts.ts'; +import { foldsExclusiveCacheTokens } from '../shared/telemetry/usage.ts'; import { defineStage, move, transform } from '@floway-dev/pipeline'; import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { GeminiContent, GeminiPayload, GeminiStreamEvent, GeminiToolGroup } from '@floway-dev/protocols/gemini'; +import type { MessagesPayload } from '@floway-dev/protocols/messages'; +import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesPayload, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; + +/** Removing a field inside a value is the same move as removing a fact, and a value that + * carried none of them comes back by identity so the rule costs nothing where it does not + * apply. */ +const withoutKeys = (value: T, keys: readonly (keyof T)[]): T => { + if (!keys.some(key => key in value)) return value; + const removed = new Set(keys as readonly string[]); + return Object.fromEntries(Object.entries(value).filter(([key]) => !removed.has(key))) as T; +}; + +/** `map`, but the array comes back by identity when every element did. Written once because + * every rule here walks a list of something — messages, input items, content parts — and + * the identity is the whole reason a layer costs only what it actually changed. */ +const mapKeepingIdentity = (items: T[], rewrite: (item: T) => T): T[] => { + let changed = false; + const mapped = items.map(item => { + const next = rewrite(item); + if (next !== item) changed = true; + return next; + }); + return changed ? mapped : items; +}; /** Role rewrites, in the fixed order `system → developer → system → user`. Later rewrites * are authoritative when flags overlap, and the last step affects only a system message - * that appears after the leading run — which is what "mid-conversation" means and why the - * fold carries a flag rather than testing the index. */ + * that appears after the leading run — which is what "mid-conversation" means. The fold + * itself is `roleRewriter`; what is here is the walk over this protocol's messages. */ export const applyRoleCompatibilityToChatCompletions = defineStage< Chat<'request.chat.chatCompletions' | 'route.attempt'>, Chat<'request.chat.chatCompletions'>, @@ -47,7 +82,7 @@ export const applyRoleCompatibilityToChatCompletions = defineStage< const rewrite = rolesFor(facts['route.attempt'].flags); if (rewrite === null) return facts; // the free do-nothing path const payload = facts['request.chat.chatCompletions']; - const messages = rewriteRoles(payload.messages, rewrite); + const messages = rewriteChatCompletionsRoles(payload.messages, rewrite); // Written conditionally the whole way down, so a conversation this does not touch // comes back by identity and the layer costs what the rewrite actually changed. return messages === payload.messages @@ -76,25 +111,41 @@ const rolesFor = (flags: readonly string[]): RoleRewrite | null => { : null; }; -type Messages = ChatCompletionsPayload['messages']; +type ChatCompletionsMessages = ChatCompletionsPayload['messages']; + +const rewriteChatCompletionsRoles = (messages: ChatCompletionsMessages, rewrite: RoleRewrite): ChatCompletionsMessages => { + const nextRole = roleRewriter(rewrite); + return mapKeepingIdentity(messages, message => { + const role = nextRole(message.role); + return role === message.role ? message : { ...message, role }; + }); +}; -const rewriteRoles = (messages: Messages, rewrite: RoleRewrite): Messages => { +/** The three rewrites in their settled order, and the state that makes "mid-conversation" + * mean what it says: the last step affects only a system message that appears after the + * leading run, which is why the fold carries a flag rather than testing the index. + * + * Written once, over roles alone. A protocol walks its own items, hands in the role it read + * off one and takes back the role to write, so the ordering exists in one place however + * differently the two protocols that use it shape a turn. An item with no role at all — + * a Responses reasoning item between two system messages — still crosses the leading run, + * which is what makes the system message after it mid-conversation. */ +const roleRewriter = (rewrite: RoleRewrite) => { let crossedLeadingSystemRun = false; - let changed = false; - const mapped = messages.map(message => { - let result = message; - if (rewrite.systemToDeveloper && result.role === 'system') result = { ...result, role: 'developer' }; - if (rewrite.developerToSystem && result.role === 'developer') result = { ...result, role: 'system' }; - if (!crossedLeadingSystemRun && result.role !== 'system') crossedLeadingSystemRun = true; - if (rewrite.midConversationSystemToUser && crossedLeadingSystemRun && result.role === 'system') { - result = { ...result, role: 'user' }; - } - if (result !== message) changed = true; + return (role: Role): Role | RewrittenRole => { + let result: Role | RewrittenRole = role; + if (rewrite.systemToDeveloper && result === 'system') result = 'developer'; + if (rewrite.developerToSystem && result === 'developer') result = 'system'; + if (!crossedLeadingSystemRun && result !== 'system') crossedLeadingSystemRun = true; + if (rewrite.midConversationSystemToUser && crossedLeadingSystemRun && result === 'system') result = 'user'; return result; - }); - return changed ? mapped : messages; + }; }; +/** The only roles the fold ever writes. Every protocol it runs on admits all three, so a + * rewritten role needs no cast to go back onto the item it came from. */ +type RewrittenRole = 'system' | 'developer' | 'user'; + /** The reasoning sentinel a forced tool choice needs. Gated by its flag, and the sentinel * is the gateway's canonical form — putting it on the wire in a vendor's shape is the * vendor normalizer's job, which is why this runs above them. */ @@ -160,12 +211,710 @@ export const stripPromptCacheKeyForChatCompletions = defineStage< request: facts => { if (!facts['route.attempt'].flags.includes('strip-prompt-cache-key')) return facts; const payload = facts['request.chat.chatCompletions']; - if (!('prompt_cache_key' in payload)) return facts; - // Removing a field inside a value is the same move as removing a fact: name it in the - // destructuring and spread the rest. - const { prompt_cache_key: dropped, ...rest } = payload; - void dropped; - return { ...facts, 'request.chat.chatCompletions': move(rest) }; + const stripped = withoutKeys(payload, ['prompt_cache_key']); + return stripped === payload ? facts : { ...facts, 'request.chat.chatCompletions': move(stripped) }; + }, + })), +}); + +// ── Messages ────────────────────────────────────────────────────────────────────────────── + +/** + * The mid-conversation system rewrite, which on this protocol is every inline system message. + * + * Anthropic's top-level `payload.system` is the only first-position system slot, so a system + * message inside `messages` is by construction past the leading run — which is why this reads + * the same flag as the other protocols' fold but is not that fold: there is no leading run to + * cross here, and no developer role to trade with. + */ +export const applyRoleCompatibilityToMessages = defineStage< + Chat<'request.chat.messages' | 'route.attempt'>, + Chat<'request.chat.messages'>, + Chat<'response.chat.messages'>, + Chat<'response.chat.messages'> +>({ + name: 'applyRoleCompatibility', + through: { + request: { + needs: ['request.chat.messages', 'route.attempt'], + consumes: [], + provides: ['request.chat.messages'], + }, + response: { needs: ['response.chat.messages'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.messages' | 'route.attempt'>, + Chat<'request.chat.messages'>, + Chat<'response.chat.messages'>, + Chat<'response.chat.messages'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('rewrite-mid-conv-system-to-user')) return facts; + const payload = facts['request.chat.messages']; + const messages = mapKeepingIdentity(payload.messages, message => + message.role === 'system' ? { role: 'user' as const, content: message.content } : message); + return messages === payload.messages + ? facts + : { ...facts, 'request.chat.messages': move({ ...payload, messages }) }; + }, + })), +}); + +/** The reasoning sentinel a forced tool choice needs, in the shape this protocol has + * natively. `tool` and `any` are the forced choices; `auto` and `none` leave the model free + * to reason. */ +export const disableReasoningOnForcedToolChoiceForMessages = defineStage< + Chat<'request.chat.messages' | 'route.attempt'>, + Chat<'request.chat.messages'>, + Chat<'response.chat.messages'>, + Chat<'response.chat.messages'> +>({ + name: 'disableReasoningOnForcedToolChoice', + through: { + request: { + needs: ['request.chat.messages', 'route.attempt'], + consumes: [], + provides: ['request.chat.messages'], + }, + response: { needs: ['response.chat.messages'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.messages' | 'route.attempt'>, + Chat<'request.chat.messages'>, + Chat<'response.chat.messages'>, + Chat<'response.chat.messages'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { + return facts; + } + const payload = facts['request.chat.messages']; + const choice = payload.tool_choice?.type; + if (choice !== 'tool' && choice !== 'any') return facts; + return { ...facts, 'request.chat.messages': move(withMessagesReasoningDisabled(payload)) }; + }, + })), +}); + +/** Only the reasoning subfield goes with the sentinel: a forced tool choice composes fine + * with structured output on these upstreams, and it is thinking it does not compose with, so + * `output_config.format` has to survive. An `output_config` that held nothing but the effort + * is nothing at all once the effort is gone, so it goes rather than riding on empty. */ +const withMessagesReasoningDisabled = (payload: MessagesPayload): MessagesPayload => { + const { output_config, ...rest } = payload; + const next: MessagesPayload = { ...rest, thinking: { type: 'disabled' as const } }; + if (output_config !== undefined) { + const remaining = withoutKeys(output_config, ['effort']); + if (Object.keys(remaining).length > 0) next.output_config = remaining; + } + return next; +}; + +/** + * Scrubs Claude Code's billing-attribution block out of the system prompt. + * + * The block carries a `cch=` that flips per call, so an upstream that reads it as + * ordinary prompt text sees a different prompt every turn and never reuses its prompt cache. + * The Claude Code subscription endpoint reads it to bill the request against the user's plan + * tier and wants it intact, which is what the flag is for: it is on for the upstreams that + * treat the block as text and off for the one that treats it as billing. + */ +export const stripBillingAttributionFromMessages = defineStage< + Chat<'request.chat.messages' | 'route.attempt'>, + Chat<'request.chat.messages'>, + Chat<'response.chat.messages'>, + Chat<'response.chat.messages'> +>({ + name: 'stripBillingAttribution', + through: { + request: { + needs: ['request.chat.messages', 'route.attempt'], + consumes: [], + provides: ['request.chat.messages'], + }, + response: { needs: ['response.chat.messages'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.messages' | 'route.attempt'>, + Chat<'request.chat.messages'>, + Chat<'response.chat.messages'>, + Chat<'response.chat.messages'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('strip-billing-attribution')) return facts; + const payload = facts['request.chat.messages']; + const system = scrubbedSystemPrompt(payload.system); + if (system === payload.system) return facts; + const withoutSystem = withoutKeys(payload, ['system']); + return { + ...facts, + 'request.chat.messages': move(system === undefined ? withoutSystem : { ...withoutSystem, system }), + }; + }, + })), +}); + +const BILLING_HEADER_LINE = /x-anthropic-billing-header[^\n]*/g; +const BILLING_CACHE_HASH = /cch=[0-9a-f]{5,};?/gi; + +const scrubText = (text: string): string => + text.replace(BILLING_HEADER_LINE, '').replace(BILLING_CACHE_HASH, '').trim(); + +/** What is left of the system prompt once the block is out of it: the value that came in when + * there was nothing to scrub, and `undefined` when nothing at all is left — an empty system + * prompt is not a system prompt, so the field goes rather than riding on empty. */ +const scrubbedSystemPrompt = (system: MessagesPayload['system']): MessagesPayload['system'] => { + if (system === undefined) return undefined; + if (typeof system === 'string') { + const scrubbed = scrubText(system); + if (scrubbed.length === 0) return undefined; + return scrubbed === system ? system : scrubbed; + } + let changed = false; + const blocks = system.flatMap(block => { + const text = scrubText(block.text); + if (text.length === 0) { changed = true; return []; } + if (text === block.text) return [block]; + changed = true; + return [{ ...block, text }]; + }); + if (!changed) return system; + return blocks.length > 0 ? blocks : undefined; +}; + +// ── Gemini ──────────────────────────────────────────────────────────────────────────────── +// +// The three strippers below are unconditional rather than flag-gated, and that is a statement +// about the target graph rather than about any upstream: Gemini has no wire of its own here, +// so every turn is served through a translation, and what no translation can carry cannot be +// sent whichever candidate answers. What was `delete ctx.payload.x` is a rewrite now — the +// record is frozen, so a stage that deleted in place would throw rather than strip. + +/** Gemini file and code parts have no equivalent anywhere the translations reach, so they go + * at source and every target sees translatable parts. A part left holding nothing goes with + * them: it is not a part any more. */ +export const stripUnsupportedPartFieldsFromGemini = defineStage< + Chat<'request.chat.gemini'>, + Chat<'request.chat.gemini'>, + Chat<'response.chat.gemini'>, + Chat<'response.chat.gemini'> +>({ + name: 'stripUnsupportedPartFields', + through: { + request: { needs: ['request.chat.gemini'], consumes: [], provides: ['request.chat.gemini'] }, + response: { needs: ['response.chat.gemini'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.gemini'>, + Chat<'request.chat.gemini'>, + Chat<'response.chat.gemini'>, + Chat<'response.chat.gemini'> + >(() => ({ + request: facts => { + const payload = facts['request.chat.gemini']; + const stripped = withUnsupportedPartFieldsStripped(payload); + return stripped === payload ? facts : { ...facts, 'request.chat.gemini': move(stripped) }; + }, + })), +}); + +const UNSUPPORTED_GEMINI_PART_FIELDS = ['fileData', 'executableCode', 'codeExecutionResult'] as const; + +const withUnsupportedPartFieldsStripped = (payload: GeminiPayload): GeminiPayload => { + const contents = payload.contents === undefined + ? undefined + : mapKeepingIdentity(payload.contents, stripContentParts); + const systemInstruction = payload.systemInstruction === undefined + ? undefined + : stripContentParts(payload.systemInstruction); + if (contents === payload.contents && systemInstruction === payload.systemInstruction) return payload; + return { + ...payload, + ...(contents === undefined ? {} : { contents }), + ...(systemInstruction === undefined ? {} : { systemInstruction }), + }; +}; + +const stripContentParts = (content: GeminiContent): GeminiContent => { + let changed = false; + const parts = content.parts.flatMap(part => { + const stripped = withoutKeys(part, UNSUPPORTED_GEMINI_PART_FIELDS); + if (stripped === part) return [part]; + changed = true; + return Object.keys(stripped).length > 0 ? [stripped] : []; + }); + return changed ? { ...content, parts } : content; +}; + +/** Only function declarations translate out of a Gemini tool group, so the rest of a group's + * capabilities go — and a group left declaring no function goes with them, because a target + * emitter offered an empty group would be offered a tool that does nothing. `tools` itself + * goes when no group survived: an empty tool list is a different request from no tools. */ +export const stripUnsupportedToolsFromGemini = defineStage< + Chat<'request.chat.gemini'>, + Chat<'request.chat.gemini'>, + Chat<'response.chat.gemini'>, + Chat<'response.chat.gemini'> +>({ + name: 'stripUnsupportedTools', + through: { + request: { needs: ['request.chat.gemini'], consumes: [], provides: ['request.chat.gemini'] }, + response: { needs: ['response.chat.gemini'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.gemini'>, + Chat<'request.chat.gemini'>, + Chat<'response.chat.gemini'>, + Chat<'response.chat.gemini'> + >(() => ({ + request: facts => { + const payload = facts['request.chat.gemini']; + const stripped = withUnsupportedToolsStripped(payload); + return stripped === payload ? facts : { ...facts, 'request.chat.gemini': move(stripped) }; }, })), }); + +const UNSUPPORTED_GEMINI_TOOL_CAPABILITIES = [ + 'googleSearch', + 'googleSearchRetrieval', + 'codeExecution', + 'computerUse', + 'urlContext', + 'fileSearch', + 'mcpServers', + 'googleMaps', +] as const; + +const withUnsupportedToolsStripped = (payload: GeminiPayload): GeminiPayload => { + const { tools } = payload; + if (tools === undefined) return payload; + let changed = false; + const kept: GeminiToolGroup[] = []; + for (const tool of tools) { + const stripped = withoutKeys(tool, UNSUPPORTED_GEMINI_TOOL_CAPABILITIES); + if (stripped !== tool) changed = true; + if (stripped.functionDeclarations !== undefined && stripped.functionDeclarations.length > 0) kept.push(stripped); + else changed = true; + } + if (kept.length === 0) return withoutKeys(payload, ['tools']); + return changed ? { ...payload, tools: kept } : payload; +}; + +/** Gemini safety controls are source-specific and have no matching control on every target + * path, so they go rather than have us pretend to enforce a policy we cannot honor + * end to end. */ +export const stripSafetySettingsFromGemini = defineStage< + Chat<'request.chat.gemini'>, + Chat<'request.chat.gemini'>, + Chat<'response.chat.gemini'>, + Chat<'response.chat.gemini'> +>({ + name: 'stripSafetySettings', + through: { + request: { needs: ['request.chat.gemini'], consumes: [], provides: ['request.chat.gemini'] }, + response: { needs: ['response.chat.gemini'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.gemini'>, + Chat<'request.chat.gemini'>, + Chat<'response.chat.gemini'>, + Chat<'response.chat.gemini'> + >(() => ({ + request: facts => { + const payload = facts['request.chat.gemini']; + const stripped = withoutKeys(payload, ['safetySettings']); + return stripped === payload ? facts : { ...facts, 'request.chat.gemini': move(stripped) }; + }, + })), +}); + +/** + * Hides Gemini thought-summary parts from a caller who did not ask for them. + * + * The opt-in is `generationConfig.thinkingConfig.includeThoughts`, which is something the + * client sent — a request-direction reading — and the rule that uses it runs on the way back. + * So it rides in this stage's own closure rather than as a declaration: a response-side + * `needs` can only name what the ending provides, which is the answer and not the turn. + */ +export const suppressThoughtPartsFromGemini = defineStage< + Chat<'request.chat.gemini'>, + Chat<'request.chat.gemini'>, + Chat<'response.chat.gemini'>, + Chat<'response.chat.gemini'> +>({ + name: 'suppressThoughtParts', + through: { + request: { needs: ['request.chat.gemini'], consumes: [], provides: [] }, + response: { needs: ['response.chat.gemini'], consumes: [], provides: ['response.chat.gemini'] }, + }, + execute: transform< + Chat<'request.chat.gemini'>, + Chat<'request.chat.gemini'>, + Chat<'response.chat.gemini'>, + Chat<'response.chat.gemini'> + >(() => { + // Assigned on the way down and read on the way back, which is the order `transform` runs + // the two halves in. + let includeThoughts!: boolean; + return { + request: facts => { + includeThoughts = facts['request.chat.gemini'].generationConfig?.thinkingConfig?.includeThoughts === true; + return facts; + }, + response: facts => { + if (includeThoughts) return facts; + const answer = facts['response.chat.gemini']; + // A refusal and a collected body have no thought parts to hide, and telling the three + // apart is reading a value rather than dispatching on a declaration. + if (isFailure(answer) || answer.kind !== 'stream') return facts; + const frames = answer.frames as AsyncIterable>; + return { + ...facts, + 'response.chat.gemini': move({ + kind: 'stream' as const, + frames: { [Symbol.asyncIterator]: () => withoutThoughtParts(frames) }, + }), + }; + }, + }; + }), +}); + +/** A candidate that has nothing left to say and has not finished is not a candidate, and an + * event left carrying no candidate, no usage, no model and no id is not an event — dropping + * both is what keeps a stream of pure thought from reaching the client as a stream of + * empties. */ +const withoutThoughtParts = async function* ( + frames: AsyncIterable>, +): AsyncGenerator> { + for await (const frame of frames) { + if (frame.type !== 'event' || 'error' in frame.event) { + yield frame; + continue; + } + + const candidates = frame.event.candidates?.flatMap(candidate => { + const parts = candidate.content.parts.filter(part => part.thought !== true); + if (!parts.length && candidate.finishReason === undefined) return []; + return [{ ...candidate, content: { ...candidate.content, parts } }]; + }); + + const event: GeminiStreamEvent = { + ...frame.event, + ...(candidates === undefined ? {} : { candidates }), + }; + if (hasGeminiEventPayload(event)) yield eventFrame(event); + } +}; + +const hasGeminiEventPayload = (event: GeminiStreamEvent): boolean => { + if ('error' in event) return true; + return (event.candidates?.length ?? 0) > 0 + || event.usageMetadata !== undefined + || event.modelVersion !== undefined + || event.responseId !== undefined; +}; + +// ── Responses ───────────────────────────────────────────────────────────────────────────── + +/** Role rewrites, in the same fixed order and by the same fold as Chat Completions; what is + * here is the walk over this protocol's input items. Only a message item carries a role, and + * an item that carries none still crosses the leading system run — a reasoning item between + * two system messages is what makes the second one mid-conversation. */ +export const applyRoleCompatibilityToResponses = defineStage< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> +>({ + name: 'applyRoleCompatibility', + through: { + request: { + needs: ['request.chat.responses', 'route.attempt'], + consumes: [], + provides: ['request.chat.responses'], + }, + response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> + >(() => ({ + request: facts => { + const rewrite = rolesFor(facts['route.attempt'].flags); + if (rewrite === null) return facts; // the free do-nothing path + // The key holds what a client may send, whose `input` is a string or a list; this chain + // runs on the canonical form the entry normalized it to, which is the one a wire takes. + const payload = facts['request.chat.responses'] as CanonicalResponsesPayload; + const nextRole = roleRewriter(rewrite); + const input = mapKeepingIdentity(payload.input, (item): ResponsesInputItem => { + if (item.type !== 'message') { + nextRole(undefined); + return item; + } + const role = nextRole(item.role); + return role === item.role ? item : { ...item, role }; + }); + return input === payload.input + ? facts + : { ...facts, 'request.chat.responses': move({ ...payload, input }) }; + }, + })), +}); + +/** The reasoning sentinel a forced tool choice needs. `required` or a named tool is forced; + * `auto` and `none` leave the model free to reason. The `reasoning` object is replaced + * rather than merged — a summary has no meaning once there is nothing to summarize — and + * the sentinel is the gateway's canonical form, which is why this runs above the vendor + * normalizers that put it on the wire in a vendor's shape. */ +export const disableReasoningOnForcedToolChoiceForResponses = defineStage< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> +>({ + name: 'disableReasoningOnForcedToolChoice', + through: { + request: { + needs: ['request.chat.responses', 'route.attempt'], + consumes: [], + provides: ['request.chat.responses'], + }, + response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { + return facts; + } + const payload = facts['request.chat.responses']; + if (!isForcedResponsesToolChoice(payload.tool_choice)) return facts; + return { ...facts, 'request.chat.responses': move({ ...payload, reasoning: { effort: 'none' } }) }; + }, + })), +}); + +const isForcedResponsesToolChoice = (choice: ResponsesPayload['tool_choice']): boolean => { + if (choice === undefined || choice === null) return false; + if (typeof choice === 'string') return choice === 'required'; + return true; +}; + +/** Drops a field the upstream would reject as an unknown argument. It runs above the vendor + * normalizers so each of them sees the already-stripped canonical payload. */ +export const stripPromptCacheKeyForResponses = defineStage< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> +>({ + name: 'stripPromptCacheKey', + through: { + request: { + needs: ['request.chat.responses', 'route.attempt'], + consumes: [], + provides: ['request.chat.responses'], + }, + response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('strip-prompt-cache-key')) return facts; + const payload = facts['request.chat.responses']; + const stripped = withoutKeys(payload, ['prompt_cache_key']); + return stripped === payload ? facts : { ...facts, 'request.chat.responses': move(stripped) }; + }, + })), +}); + +/** + * Restores OpenAI's inclusive input-token contract on an upstream that reports the cache + * buckets alongside `input_tokens` instead of inside it. + * + * Unconditional on this chain rather than flag-gated: `foldsExclusiveCacheTokens` owns the + * decision and reads `total_tokens` as the witness, and the `usage-exclusive-cached-tokens` + * flag is a declaration input for the responses whose totals witness nothing. Every event + * that carries a response resource repeats the whole resource, so the rewrite applies to each + * of them rather than to a single terminal frame. The evidence for the two conventions and + * the contradictions that raise are documented at `foldsExclusiveCacheTokens` and at the Chat + * Completions normalizer of the same name. + */ +export const normalizeExclusiveCachedTokensForResponses = defineStage< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> +>({ + name: 'normalizeExclusiveCachedTokens', + through: { + request: { needs: ['route.attempt'], consumes: [], provides: [] }, + response: { needs: ['response.chat.responses'], consumes: [], provides: ['response.chat.responses'] }, + }, + execute: transform< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> + >(() => { + // Which attempt this is gets read on the way down and spoken about on the way back, which + // is the order `transform` runs the two halves in. + let attempt!: AttemptSelector; + return { + request: facts => { + attempt = facts['route.attempt']; + return facts; + }, + response: facts => { + const answer = facts['response.chat.responses']; + if (isFailure(answer) || answer.kind !== 'stream') return facts; + const frames = answer.frames as AsyncIterable>; + const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); + const identity = `${attempt.upstreamId}/${attempt.modelId}`; + return { + ...facts, + 'response.chat.responses': move({ + kind: 'stream' as const, + frames: { [Symbol.asyncIterator]: () => withCacheTokensFolded(frames, declaredExclusive, identity) }, + }), + }; + }, + }; + }), +}); + +const withCacheTokensFolded = async function* ( + frames: AsyncIterable>, + declaredExclusive: boolean, + identity: string, +): AsyncGenerator> { + for await (const frame of frames) { + if (frame.type !== 'event') { + yield frame; + continue; + } + const event = foldInboundUsage(frame.event, declaredExclusive, identity); + yield event === frame.event ? frame : eventFrame(event); + } +}; + +/** Responses carries usage on `event.response.usage` and names the buckets `input_tokens` and + * `input_tokens_details.{cached_tokens, cache_write_tokens}`. */ +const foldInboundUsage = ( + event: ResponsesStreamEvent, + declaredExclusive: boolean, + identity: string, +): ResponsesStreamEvent => { + if (!('response' in event)) return event; + const response = asJsonObject(event.response); + const usage = asJsonObject(response?.usage); + if (!response || !usage) return event; + const inputTokens = readJsonNumber(usage.input_tokens); + const outputTokens = readJsonNumber(usage.output_tokens); + if (inputTokens == null || outputTokens == null) return event; + + const details = asJsonObject(usage.input_tokens_details); + const cacheRead = readJsonNumber(details?.cached_tokens) ?? 0; + const cacheWrite = readJsonNumber(details?.cache_write_tokens) ?? 0; + if (cacheRead === 0 && cacheWrite === 0) return event; + + const fold = foldsExclusiveCacheTokens(declaredExclusive, { + inputTokens, + outputTokens, + totalTokens: readJsonNumber(usage.total_tokens) ?? undefined, + cacheRead, + cacheWrite, + }, identity); + if (!fold) return event; + + const nextUsage: JsonObject = { ...usage, input_tokens: inputTokens + cacheRead + cacheWrite }; + const nextResponse: JsonObject = { ...response, usage: nextUsage }; + return { ...event, response: nextResponse } as unknown as ResponsesStreamEvent; +}; + +/** DeepSeek says "no reasoning" with a top-level `thinking: { type: 'disabled' }` rather than + * with the canonical sentinel, so the sentinel is put into its wire form here — last among + * this chain's rewrites, because a vendor normalizer has the final say on the outbound body. + * https://api-docs.deepseek.com/zh-cn/guides/thinking_mode */ +export const vendorDeepSeekNormalizeForResponses = defineStage< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> +>({ + name: 'vendorDeepSeekNormalize', + through: { + request: { + needs: ['request.chat.responses', 'route.attempt'], + consumes: [], + provides: ['request.chat.responses'], + }, + response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('vendor-deepseek')) return facts; + const payload = facts['request.chat.responses']; + if (payload.reasoning?.effort !== 'none') return facts; + const { reasoning, ...rest } = payload; + const normalized: ResponsesPayloadWithDeepSeekThinking = { ...rest, thinking: { type: 'disabled' } }; + return { ...facts, 'request.chat.responses': move(normalized) }; + }, + })), +}); + +/** Qwen says the same thing with a top-level `enable_thinking: false`. + * https://www.alibabacloud.com/help/en/model-studio/deep-thinking */ +export const vendorQwenNormalizeForResponses = defineStage< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> +>({ + name: 'vendorQwenNormalize', + through: { + request: { + needs: ['request.chat.responses', 'route.attempt'], + consumes: [], + provides: ['request.chat.responses'], + }, + response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.responses' | 'route.attempt'>, + Chat<'request.chat.responses'>, + Chat<'response.chat.responses'>, + Chat<'response.chat.responses'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('vendor-qwen')) return facts; + const payload = facts['request.chat.responses']; + if (payload.reasoning?.effort !== 'none') return facts; + const { reasoning, ...rest } = payload; + const normalized: ResponsesPayloadWithQwenThinking = { ...rest, enable_thinking: false }; + return { ...facts, 'request.chat.responses': move(normalized) }; + }, + })), +}); + +/** Neither field is a Responses field, so each is declared beside the vendor that reads it + * rather than widening the protocol's own request type. */ +type ResponsesPayloadWithDeepSeekThinking = Omit & { thinking: { type: 'disabled' } }; +type ResponsesPayloadWithQwenThinking = Omit & { enable_thinking: false }; From 13f2a9e04c93b375d3c459ff06642a05bef2a603 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 06:29:20 +0800 Subject: [PATCH 028/156] test(chat): assemble each family's interceptor array in the order its chain runs it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-rule tests say what one stage does; nothing said that the array a family's `pipeline.ts` will hold assembles at all. `compose` refuses declarations that do not line up, so running the whole array between stages that declare what the real neighbours declare — an edge that needs the answer, the headers and the billed set on the way up, and an ending that answers with all three — is that check. It also pins the one ordering that lives between stages rather than inside one: the reasoning sentinel is the gateway's canonical form, and a vendor normalizer can only put it on the wire in the vendor's shape because it runs after the stage that wrote it. --- .../data-plane/chat-interceptors_test.ts | 142 ++++++++++++++++-- 1 file changed, 130 insertions(+), 12 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index 7b4619b446..012b40851e 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -38,32 +38,59 @@ import type { ResponsesInputItem, ResponsesPayload, ResponsesStreamEvent } from const attemptWith = (...flags: string[]): AttemptSelector => ({ upstreamId: 'u', modelId: 'm', flags }); /** - * Runs one stage and hands back both directions: the record it passed down, and the record - * that came back up. + * Runs a family's whole interceptor array, in the order its chain runs it, between stages + * that declare what its real neighbours declare: an edge that needs the answer, the headers + * and the billed set on the way up, and an ending that answers with all three. * - * The sink has to answer at the key the stage above declared it needs, or `compose` refuses - * the array — that refusal is the assembly check doing its job, and is why the harness is - * told which key a protocol answers at rather than guessing one. + * `compose` refuses an array whose declarations do not line up, so an array that runs here is + * one that family's `pipeline.ts` can hold — which is the whole of what these harnesses check + * beyond the rewrites themselves. */ -const runStage = async ( - stage: unknown, +const runChain = async ( responseKey: string, + stages: readonly unknown[], facts: Record, answer: unknown = { kind: 'value' as const, body: null }, ): Promise<{ readonly down: Record; readonly up: Record }> => { let down: Record = {}; - const sink = defineStage, Record>({ - name: 'sink', - return: { provides: [responseKey] }, + const edge = defineStage< + Record, + Record, + Record, + Record + >({ + name: 'edge', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: [responseKey, 'response.http.headers', 'response.usage.billable'], + consumes: [], + provides: [], + }, + }, + execute: async (received, next) => await next(received), + }); + const ending = defineStage, Record>({ + name: 'ending', + return: { provides: [responseKey, 'response.http.headers', 'response.usage.billable'] }, execute: async received => { down = received; - return move({ ...received, [responseKey]: answer }); + return move({ ...received, [responseKey]: answer, 'response.http.headers': [], 'response.usage.billable': [] }); }, }); - const { facts: up } = await run(compose('one', [stage as never, sink]), move(facts) as never, {}); + const { facts: up } = await run(compose('chain', [edge, ...(stages as never[]), ending]), move(facts) as never, {}); return { down, up: up as Record }; }; +/** One stage is the same harness with an array of one. */ +const runStage = async ( + stage: unknown, + responseKey: string, + facts: Record, + answer: unknown = { kind: 'value' as const, body: null }, +): Promise<{ readonly down: Record; readonly up: Record }> => + await runChain(responseKey, [stage], facts, answer); + /** An answer in the shape a wire hands up: frames, read once. */ const streamOf = (frames: readonly ProtocolFrame[]) => ({ kind: 'stream' as const, @@ -549,3 +576,94 @@ const usageEvent = (usage: Record): ProtocolFrame): Record => (frame as { event: { response: { usage: Record } } }).event.response.usage; + +// Each array below is the order that family's chain runs, and running it is what says the +// order is a property of the array rather than of any one stage: assembly refuses +// declarations that do not line up, and a rewrite that had to see another rewrite's output +// only sees it from where it sits. +describe('a family\'s interceptor array, in the order its chain runs it', () => { + it('assembles and runs the Messages array', async () => { + const { down } = await runChain('response.chat.messages', [ + stripBillingAttributionFromMessages, + disableReasoningOnForcedToolChoiceForMessages, + applyRoleCompatibilityToMessages, + ], { + 'request.chat.messages': messagesPayload([{ role: 'system', content: 'inline' }], { + system: 'x-anthropic-billing-header: cch=abcdef12\nreal instructions', + tool_choice: { type: 'any' }, + }), + 'route.attempt': attemptWith( + 'strip-billing-attribution', + 'disable-reasoning-on-forced-tool-choice', + 'rewrite-mid-conv-system-to-user', + ), + }); + const after = down['request.chat.messages'] as MessagesPayload; + expect(after.system).toBe('real instructions'); + expect(after.thinking).toEqual({ type: 'disabled' }); + expect(after.messages.map(message => message.role)).toEqual(['user']); + }); + + it('assembles and runs the Gemini array', async () => { + const answer = streamOf([ + eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }, { text: 'the answer' }] }, finishReason: 'STOP' }] }), + ]); + const { down, up } = await runChain('response.chat.gemini', [ + stripUnsupportedPartFieldsFromGemini, + stripUnsupportedToolsFromGemini, + stripSafetySettingsFromGemini, + suppressThoughtPartsFromGemini, + ], { + 'request.chat.gemini': { + contents: [{ role: 'user', parts: [{ text: 'hi', fileData: { mimeType: 'text/plain', fileUri: 'u' } }] }], + tools: [{ googleSearch: {} }], + safetySettings: [{ category: 'c', threshold: 't' }], + } satisfies GeminiPayload, + }, answer); + const after = down['request.chat.gemini'] as GeminiPayload; + expect(after.contents?.[0]?.parts).toEqual([{ text: 'hi' }]); + expect('tools' in after).toBe(false); + expect('safetySettings' in after).toBe(false); + const frames = await drain(up['response.chat.gemini']); + const event = (frames[0] as { event: Extract }).event; + expect(event.candidates?.[0]?.content.parts).toEqual([{ text: 'the answer' }]); + }); + + // The one ordering between stages rather than inside one: the sentinel is the gateway's + // canonical form, and a vendor normalizer can only put it on the wire in the vendor's shape + // because it runs after the stage that wrote it. + it('assembles and runs the Responses array, the vendor normalizers last', async () => { + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); + const { down, up } = await runChain('response.chat.responses', [ + disableReasoningOnForcedToolChoiceForResponses, + applyRoleCompatibilityToResponses, + stripPromptCacheKeyForResponses, + normalizeExclusiveCachedTokensForResponses, + vendorDeepSeekNormalizeForResponses, + vendorQwenNormalizeForResponses, + ], { + 'request.chat.responses': responsesPayload([ + { type: 'message', role: 'system', content: 'lead' }, + { type: 'message', role: 'user', content: 'hi' }, + { type: 'message', role: 'system', content: 'mid' }, + ], { tool_choice: 'required', prompt_cache_key: 'k' }), + 'route.attempt': attemptWith( + 'disable-reasoning-on-forced-tool-choice', + 'rewrite-mid-conv-system-to-user', + 'strip-prompt-cache-key', + 'vendor-deepseek', + ), + }, answer); + const after = down['request.chat.responses'] as Record; + expect(rolesOfPayload(after)).toEqual(['system', 'user', 'user']); + expect('prompt_cache_key' in after).toBe(false); + // The sentinel this chain wrote reached DeepSeek's normalizer and left in DeepSeek's shape. + expect('reasoning' in after).toBe(false); + expect(after.thinking).toEqual({ type: 'disabled' }); + const frames = await drain(up['response.chat.responses']); + expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 150 }); + }); +}); + +const rolesOfPayload = (payload: Record): unknown[] => + (payload.input as ResponsesInputItem[]).map(item => (item as { role?: unknown }).role); From d10b06c0ab0044fefebc1ea3f3c721f582cc0925 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 06:36:24 +0800 Subject: [PATCH 029/156] feat(chat): run each family's interceptors as the stages they became MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three chains ran nothing between the fork and the ending: their rules were still in the interceptor form, so Messages sent no role rewrite, Gemini stripped no unsupported field and suppressed no thought part — the last reaching a client that never asked to see one. Each rule is now composed where array position states when it runs, and the order within a family is the order the onion applied them in. --- .../gateway/src/data-plane/chat/gemini/pipeline.ts | 13 ++++++++++--- .../src/data-plane/chat/messages/pipeline.ts | 8 ++++++++ .../src/data-plane/chat/responses/pipeline.ts | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 44c2f7830e..a2980f2607 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -50,6 +50,12 @@ import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts' import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import { billableUsageFromChatCompletionsEvent } from '../chat-completions/usage.ts'; import type { ChatFacts } from '../facts.ts'; +import { + stripSafetySettingsFromGemini, + stripUnsupportedPartFieldsFromGemini, + stripUnsupportedToolsFromGemini, + suppressThoughtPartsFromGemini, +} from '../interceptors.ts'; import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; @@ -288,9 +294,6 @@ const meterChatCompletions = ( ): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { let settle!: (outcome: StreamOutcome) => void; const outcome = new Promise(resolve => { settle = resolve; }); - // Running out without the terminal frame is what "it did not finish" means, and it is known - // at the same moment the usage is. - const sawTerminal = false; const generator = (async function* () { let reported: BillableUsage | undefined; try { @@ -390,5 +393,9 @@ export const geminiServePipeline = (payload: GeminiPayload): Pipeline Date: Mon, 17 Aug 2026 06:42:40 +0800 Subject: [PATCH 030/156] feat(messages): answer the Claude Code probe as a stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe is a turn the gateway can answer truthfully without spending an upstream one, and it was the last rule in this family still only in the interceptor form — so a pipelined /v1/messages dialled an upstream for it and, on a Messages-via-Responses candidate, took the hard 400 that OpenAI's floor on max_output_tokens produces. It answers rather than descending, so it carries the return trait, and it lives beside the ending because what it answers with is this family's own keys. It provides the streamed-usage key as null: a run that never opened a stream has nothing still to read, and settlement would otherwise wait for numbers that never come. The recognition and the frames stay where they were and are shared, because the interceptor path is still reached while Responses translates through it. --- .../interceptors/answer-claude-code-probe.ts | 4 +- .../src/data-plane/chat/messages/pipeline.ts | 72 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/messages/interceptors/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/messages/interceptors/answer-claude-code-probe.ts index 928cec19b0..4f1f8d230e 100644 --- a/packages/gateway/src/data-plane/chat/messages/interceptors/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/messages/interceptors/answer-claude-code-probe.ts @@ -74,7 +74,7 @@ const soleUserPromptOf = (payload: MessagesPayload): string | null => { return block.type === 'text' ? block.text : null; }; -const isClaudeCodeProbe = (payload: MessagesPayload, headers: Headers): boolean => { +export const isClaudeCodeProbe = (payload: MessagesPayload, headers: Headers): boolean => { if (payload.max_tokens !== 1) return false; // Every real Claude Code turn ships the session's tools; the one-token // probes never do. @@ -90,7 +90,7 @@ const isClaudeCodeProbe = (payload: MessagesPayload, headers: Headers): boolean // usage block is load-bearing — the validation path dereferences // `usage.input_tokens` / `usage.output_tokens` unconditionally, and a missing // `usage` throws inside the CLI and reads as a failed probe. -const probeFrames = async function* (model: string): AsyncGenerator> { +export const probeFrames = async function* (model: string): AsyncGenerator> { yield eventFrame({ type: 'message_start', message: { diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index 12cb07c399..6921ce5f3c 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -36,6 +36,7 @@ import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { isClaudeCodeProbe, probeFrames } from './interceptors/answer-claude-code-probe.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import { renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { @@ -153,6 +154,76 @@ const renderSSE = (frames: AsyncIterable>): A * key — which is what will make it interchangeable with a translated chain: both hand up * `response.chat.messages`, and the stage above cannot tell which ran. */ +/** + * Answers Claude Code's one-token probe itself, rather than spending an upstream turn on it. + * + * The CLI asks "is this model usable?" by generating one token against it and reporting the + * model unusable if the call throws. A one-token cap is not portable — OpenAI's Responses API + * floors `max_output_tokens` at 16 and rejects anything lower with a hard 400 — so every + * Messages-via-Responses candidate fails a probe that is asking nothing this gateway cannot + * answer. Resolution has already picked a real candidate by the time this runs, so an id no + * upstream serves still fails above with a 404; what is suppressed is only the generation. + * + * It answers rather than descending, which is why it carries the `return` trait — and why it + * has to live here rather than beside the shared interceptors: what it answers with is this + * family's own response keys. + */ +const answerClaudeCodeProbe = defineStage< + M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers'>, + M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers'>, + M<'response.chat.messages' | 'response.chat.messages.streamedUsage' + | 'response.usage.billable' | 'response.http.headers'>, + M<'response.chat.messages' | 'response.chat.messages.streamedUsage' + | 'response.usage.billable' | 'response.http.headers'>, + M<'response.chat.messages' | 'response.chat.messages.streamedUsage' + | 'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'answerClaudeCodeProbe', + through: { + request: { + needs: ['request.chat.messages', 'route.attempt', 'ingress.http.headers'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.chat.messages', 'response.http.headers'], + consumes: [], + provides: [], + }, + }, + return: { + provides: [ + 'response.chat.messages', + 'response.chat.messages.streamedUsage', + 'response.usage.billable', + 'response.http.headers', + ], + }, + execute: async (facts, next, use) => { + const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); + if (!isClaudeCodeProbe(facts['request.chat.messages'], headers)) return await next(facts); + + const candidate = use.resolveAttempt(facts['route.attempt']); + use.log.debug('answering a Claude Code probe without dialling', { upstream: facts['route.attempt'].upstreamId }); + return move({ + ...facts, + 'response.chat.messages': { + kind: 'stream' as const, + frames: probeFrames(facts['request.chat.messages'].model), + }, + // Nothing streams from an upstream here, so there is nothing still to read: the row is + // written now, at zero, which keeps the request visible without inventing latency. + 'response.chat.messages.streamedUsage': null, + 'response.usage.billable': [{ + identity: telemetryModelIdentity(candidate, providerModelOf(candidate).id), + quantities: {}, + }], + 'response.http.headers': [], + }); + }, +}); + const callMessagesUpstream = defineStage< M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers'>, M<'response.chat.messages' | 'response.chat.messages.streamedUsage' @@ -359,6 +430,7 @@ export const messagesServePipeline = (payload: MessagesPayload): Pipeline Date: Mon, 17 Aug 2026 06:49:56 +0800 Subject: [PATCH 031/156] feat(chat): hand a turn's own state back to the client that will carry it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every chain read affinity on the way down and none wrote it on the way back, so a turn produced no carrier and the follow-up that quoted it could not be pinned to the upstream that issued it — the half that makes client-carried state mean anything was simply absent. It belongs at the edge, on the frames themselves: below the fold there would be nothing left to rewrite. The Claude Code probe selects its candidate for the same reason a dialled attempt does — a turn answered for a candidate is one a follow-up must come back to. --- .../chat/chat-completions/pipeline.ts | 16 +++++++++++++--- .../src/data-plane/chat/gemini/pipeline.ts | 16 +++++++++++++--- .../src/data-plane/chat/messages/pipeline.ts | 18 ++++++++++++++++-- .../src/data-plane/chat/responses/pipeline.ts | 15 +++++++++++++-- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 54fa336a71..db62a8610c 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -12,6 +12,7 @@ // native wire is here; the translated ones hand up the same key, which is what will make // them interchangeable with it. +import { wrapChatCompletionsAffinityEgress } from './affinity/egress.ts'; import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; import { billableUsageFromChatCompletionsEvent } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; @@ -29,6 +30,7 @@ import { disableReasoningOnForcedToolChoiceForChatCompletions, stripPromptCacheKeyForChatCompletions, } from '../interceptors.ts'; +import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; @@ -74,7 +76,8 @@ const emitChatCompletions = defineStage< C<'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk'>, C<'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk' | 'response.chat.chatCompletions' | 'response.http.headers'>, - C<'response.chat.chatCompletions.rendered' | 'response.http.status' | 'response.http.headers'> + C<'response.chat.chatCompletions.rendered' | 'response.http.status' | 'response.http.headers'>, + ChatServices >({ name: 'emitChatCompletions', through: { @@ -89,7 +92,7 @@ const emitChatCompletions = defineStage< provides: ['response.chat.chatCompletions.rendered', 'response.http.status', 'response.http.headers'], }, }, - execute: async (facts, next) => { + execute: async (facts, next, use) => { const back = await next(facts); const { 'response.chat.chatCompletions': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what @@ -118,7 +121,14 @@ const emitChatCompletions = defineStage< }; } - const frames = answer.frames as AsyncIterable>; + // The turn's own state, written back into the frames the client is handed: a follow-up + // carrying it comes back to the upstream that issued it. This is the other half of the + // affinity the resolver read on the way down, and it has to sit here because it rewrites + // the frames — below the fold, and there would be nothing left to rewrite. + const frames = wrapChatCompletionsAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ); if (!back['ingress.chat.chatCompletions.wantsStream']) { return { ...rest, diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index a2980f2607..ab43eeb4a1 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -59,6 +59,8 @@ import { import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; +import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; +import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; @@ -103,7 +105,8 @@ const emitGemini = defineStage< C<'ingress.chat.gemini.wantsStream'>, C<'ingress.chat.gemini.wantsStream'>, C<'ingress.chat.gemini.wantsStream' | 'response.chat.gemini' | 'response.http.headers'>, - C<'response.chat.gemini.rendered' | 'response.http.status' | 'response.http.headers'> + C<'response.chat.gemini.rendered' | 'response.http.status' | 'response.http.headers'>, + ChatServices >({ name: 'emitGemini', through: { @@ -118,7 +121,7 @@ const emitGemini = defineStage< provides: ['response.chat.gemini.rendered', 'response.http.status', 'response.http.headers'], }, }, - execute: async (facts, next) => { + execute: async (facts, next, use) => { const back = await next(facts); const { 'response.chat.gemini': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what @@ -144,7 +147,14 @@ const emitGemini = defineStage< }; } - const frames = answer.frames as AsyncIterable>; + // The turn's own state, written back into the frames the client is handed: a follow-up + // carrying it comes back to the upstream that issued it. This is the other half of the + // affinity the resolver read on the way down, and it has to sit here because it rewrites + // the frames — below the fold, and there would be nothing left to rewrite. + const frames = wrapGeminiAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ); if (!back['ingress.chat.gemini.wantsStream']) { return { ...rest, diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index 6921ce5f3c..012122b528 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -35,6 +35,8 @@ import { import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; +import { wrapMessagesAffinityEgress } from './affinity/egress.ts'; +import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { isClaudeCodeProbe, probeFrames } from './interceptors/answer-claude-code-probe.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; @@ -80,6 +82,8 @@ const emitMessages = defineStage< M<'ingress.chat.messages.wantsStream'>, M<'ingress.chat.messages.wantsStream' | 'response.chat.messages' | 'response.http.headers'>, M<'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers'> + , + ChatServices >({ name: 'emitMessages', through: { @@ -90,7 +94,7 @@ const emitMessages = defineStage< provides: ['response.chat.messages.rendered', 'response.http.status', 'response.http.headers'], }, }, - execute: async (facts, next) => { + execute: async (facts, next, use) => { const back = await next(facts); const { 'response.chat.messages': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what @@ -116,7 +120,14 @@ const emitMessages = defineStage< }; } - const frames = answer.frames as AsyncIterable>; + // The turn's own state, written back into the frames the client is handed: a follow-up + // carrying it comes back to the upstream that issued it. This is the other half of the + // affinity the resolver read on the way down, and it has to sit here because it rewrites + // the frames — below the fold, and there would be nothing left to rewrite. + const frames = wrapMessagesAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ); if (!back['ingress.chat.messages.wantsStream']) { return { ...rest, @@ -205,6 +216,9 @@ const answerClaudeCodeProbe = defineStage< if (!isClaudeCodeProbe(facts['request.chat.messages'], headers)) return await next(facts); const candidate = use.resolveAttempt(facts['route.attempt']); + // The probe is answered *for* this candidate, so it is the one a follow-up turn carrying + // our own state must come back to — the same statement a dialled attempt makes. + use.selectAffinity(candidate); use.log.debug('answering a Claude Code probe without dialling', { upstream: facts['route.attempt'].upstreamId }); return move({ ...facts, diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index fd4d37c636..a2664b5134 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -57,6 +57,8 @@ import { import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; +import { wrapResponsesAffinityEgress } from './affinity/egress.ts'; +import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import { doneFrame, renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; @@ -101,6 +103,8 @@ const emitResponses = defineStage< R<'ingress.chat.responses.wantsStream'>, R<'ingress.chat.responses.wantsStream' | 'response.chat.responses' | 'response.http.headers'>, R<'response.chat.responses.rendered' | 'response.http.status' | 'response.http.headers'> + , + ChatServices >({ name: 'emitResponses', through: { @@ -111,7 +115,7 @@ const emitResponses = defineStage< provides: ['response.chat.responses.rendered', 'response.http.status', 'response.http.headers'], }, }, - execute: async (facts, next) => { + execute: async (facts, next, use) => { const back = await next(facts); const { 'response.chat.responses': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what @@ -142,7 +146,14 @@ const emitResponses = defineStage< }; } - const frames = answer.frames as AsyncIterable>; + // The turn's own state, written back into the frames the client is handed: a follow-up + // carrying it comes back to the upstream that issued it. This is the other half of the + // affinity the resolver read on the way down, and it has to sit here because it rewrites + // the frames — below the fold, and there would be nothing left to rewrite. + const frames = wrapResponsesAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ); if (!back['ingress.chat.responses.wantsStream']) { return { ...rest, From 416a9abb936643a8e923879ab044dadf6f58f6d4 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 07:09:28 +0800 Subject: [PATCH 032/156] test(chat): read each family's frames as the affinity carrier leaves them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every chain test described a turn from before the edge handed client-carried state back, so all four read a stream the chain no longer writes: an extra Chat Completions chunk, a redacted_thinking block holding Messages content block 0, a thoughtSignature on the Gemini Part that answered, and a reasoning item ahead of the Responses answer. The exact-sequence assertions stay exact and filter the carrier out, so what they still state is what the protocol itself answered with — including where the carrier moved the rest of it. Messages blocks answer one index along, the Responses terminal event is numbered behind two frames of its own, and the Gemini fold keeps the two Parts the wire below sent, because merging a signed Part into its neighbour would spread that signature over text it does not cover. Each family also states the property the egress exists for, which no chain test covered: the carrier is written, and it decrypts to the upstream this run dialled. `selectAffinity` is wired to the affinity context the way the prologue wires it, so the upstream the carrier names is the one the chain chose rather than the mock's default. --- .../chat/chatcompletions-pipeline_test.ts | 41 +++++++++-- .../data-plane/chat/gemini-pipeline_test.ts | 62 ++++++++++++++-- .../data-plane/chat/messages-pipeline_test.ts | 72 ++++++++++++++++--- .../chat/responses-pipeline_test.ts | 70 +++++++++++++++--- 4 files changed, 215 insertions(+), 30 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index 4ef9e535d2..145318299a 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -1,7 +1,8 @@ // The Chat Completions chain, run. Assembly is checked where every family's is; what is // written down here is what only running it can say — that the edge folds a stream into one -// object when the client did not ask to stream, that a refusal keeps the upstream's own -// status and words, and that a dial nobody answered is a value the fork can move past. +// object when the client did not ask to stream, that it hands the turn's own state back for +// the client to carry, that a refusal keeps the upstream's own status and words, and that a +// dial nobody answered is a value the fork can move past. import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -56,6 +57,13 @@ const chunk = (text: string): ChatCompletionsStreamEvent => ({ choices: [{ index: 0, delta: { content: text }, finish_reason: null }], }); +/** What this frame carries of the turn's own state, if it is the carrier the edge writes: a + * chunk whose delta holds nothing but `reasoning_opaque`. */ +const affinityCarrier = (frame: SseFrame): string | undefined => { + if (frame.data === '[DONE]') return undefined; + return (JSON.parse(frame.data) as ChatCompletionsStreamEvent).choices[0]?.delta.reasoning_opaque ?? undefined; +}; + /** A stream that stops without ever saying it ended — what a dropped upstream looks like. */ const truncated = (...events: readonly ChatCompletionsStreamEvent[]): ProviderStreamResult => ({ ok: true, @@ -100,7 +108,9 @@ const serveWith = async (gateway: ReturnType, wantsSt rememberCandidates: () => {}, rememberChatSelection: () => {}, chatPayloadFor: () => affinityPayload, - selectAffinity: () => {}, + // Wired where the app wires it: the carrier the edge writes is addressed to whatever the + // chain named here. + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); @@ -163,13 +173,36 @@ describe('the chat completions chain', () => { for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); expect(facts['response.http.status']).toBe(200); - expect(frames.map(frame => frame.data)).toEqual([ + // The carrier the edge writes has a test of its own below; what the protocol itself + // answered with is these frames, in this order. + expect(frames.filter(frame => affinityCarrier(frame) === undefined).map(frame => frame.data)).toEqual([ JSON.stringify(chunk('he')), JSON.stringify(chunk('llo')), '[DONE]', ]); }); + // The other half of affinity: the resolver reads client-carried state on the way down, and + // this is what the client is given to carry back. A turn that handed back nothing would + // leave the follow-up quoting it with no way to name the upstream that issued it. + it('hands the turn-s own state back on a chunk of its own', async () => { + resolves([candidate(async () => stream(chunk('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + const frames: SseFrame[] = []; + for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); + const carried = frames.flatMap(frame => affinityCarrier(frame) ?? []); + + expect(carried).toHaveLength(1); + // It names the upstream that answered, sealed under this run's own secret — which is what + // lets the next turn be pinned to it. + expect(await gateway.affinity.codec.unwrap(carried[0]!, 'chat-completions.reasoning_opaque')).toMatchObject({ + kind: 'owned', + affinity: { upstreamId: 'up_a', modelId: 'chat-model' }, + }); + }); + // The upstream speaks SSE whatever the client asked for, so a client that did not ask to // stream is answered from the same frames — folded here rather than read a second time. it('folds the frames into one object when the client did not', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index 8286c21917..3b75f52d16 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -1,7 +1,8 @@ // Gemini's pipeline, assembled and run. Gemini is the source-only family and the one with no // wire of its own, so what a run of it has to show beyond the entry contract is the round -// trip: a turn translated out to Chat Completions, an answer translated back, and the reading -// taken on the dialect the upstream actually spoke rather than on the one the client did. +// trip: a turn translated out to Chat Completions, an answer translated back, the turn's own +// state handed back on a Part for the client to carry, and the reading taken on the dialect +// the upstream actually spoke rather than on the one the client did. import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -12,7 +13,7 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; -import { GEMINI_MISSING_TERMINAL_MESSAGE, type GeminiPayload } from '@floway-dev/protocols/gemini'; +import { GEMINI_MISSING_TERMINAL_MESSAGE, type GeminiPayload, type GeminiResult } from '@floway-dev/protocols/gemini'; import { directFetcher, type ModelCandidate, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -141,7 +142,9 @@ const serveWith = async (gateway: ReturnType, facts: rememberCandidates: () => {}, rememberChatSelection: () => {}, chatPayloadFor: () => { asked += 1; return affinityPayload; }, - selectAffinity: (candidate: ModelCandidate) => { selected.push(candidate); }, + // Wired where the app wires it: the carrier the edge writes is addressed to whatever the + // chain named here. + selectAffinity: (answered: ModelCandidate) => { selected.push(answered); gateway.affinity.select(answered); }, resolveAttempt, } as never, ); @@ -161,6 +164,27 @@ const collect = async (rendered: unknown): Promise => { return frames; }; +/** The turn's own answer. The carrier the edge writes back rides as a `thoughtSignature` on + * a Part rather than as an event of its own, so taking it off leaves what Gemini itself + * said. */ +const withoutCarrier = (event: GeminiResult): GeminiResult => ({ + ...event, + ...(event.candidates === undefined ? {} : { + candidates: event.candidates.map(candidate => ({ + ...candidate, + content: { + ...candidate.content, + parts: candidate.content.parts.map(({ thoughtSignature: _carried, ...part }) => part), + }, + })), + }), +}); + +/** Every piece of this turn's own state the client was handed back. */ +const affinityCarriers = (events: readonly GeminiResult[]): readonly string[] => + events.flatMap(event => (event.candidates ?? []).flatMap(candidate => + candidate.content.parts.flatMap(part => part.thoughtSignature ?? []))); + describe('the gemini pipeline', () => { // The whole entry contract, and it does not mention `request.chat.gemini`: the turn the // ending translates is not the one the caller handed in but the one materializeAttempt put @@ -183,7 +207,8 @@ describe('the gemini pipeline', () => { // The answer comes back before the drain runs, which is what lets a streaming family // hand its stream on: the frames are still there to read. - expect((await collect(facts['response.chat.gemini.rendered'])).map(frame => JSON.parse(frame.data) as unknown)).toEqual([ + expect((await collect(facts['response.chat.gemini.rendered'])) + .map(frame => withoutCarrier(JSON.parse(frame.data) as GeminiResult))).toEqual([ { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'he' }] } }] }, { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'llo' }] }, finishReason: 'STOP' }], @@ -229,14 +254,37 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts({ 'ingress.chat.gemini.wantsStream': false })); - expect(facts['response.chat.gemini.rendered']).toEqual({ - candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'hello' }] }, finishReason: 'STOP' }], + // The two chunks stay two Parts: the carrier signs the Part it rides on, and merging a + // signed Part into its neighbour would spread that signature over text it does not cover. + expect(withoutCarrier(facts['response.chat.gemini.rendered'] as GeminiResult)).toEqual({ + candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'he' }, { text: 'llo' }] }, finishReason: 'STOP' }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 7, totalTokenCount: 12 }, }); expect(facts['response.http.status']).toBe(200); await drain(); }); + // The other half of affinity: the resolver reads client-carried state on the way down, and + // this is what the client is given to carry back. A turn that handed back nothing would + // leave the follow-up quoting it with no way to name the upstream that issued it. + it('hands the turn-s own state back on the Part that answered', async () => { + resolves([candidate('up_a', async () => streamed(turn()))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts, drain } = await serveWith(gateway, entryFacts()); + const frames = await collect(facts['response.chat.gemini.rendered']); + const carried = affinityCarriers(frames.map(frame => JSON.parse(frame.data) as GeminiResult)); + + expect(carried).toHaveLength(1); + // It names the upstream that answered, sealed under this run's own secret — which is what + // lets the next turn be pinned to it. + expect(await gateway.affinity.codec.unwrap(carried[0]!, 'gemini.part.thoughtSignature')).toMatchObject({ + kind: 'owned', + affinity: { upstreamId: 'up_a', modelId: 'gemini-2.5-pro' }, + }); + await drain(); + }); + it('fails a refusal over to the next candidate, and answers in the last upstream-s own words', async () => { const tried: string[] = []; resolves([ diff --git a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts index 1b874177b3..e9bd8ed991 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts @@ -1,10 +1,10 @@ // The Messages chain, run. Assembly is checked where every family's is; what is written // down here is what only running it can say — that the edge writes Anthropic's own named // SSE events when the client asked to stream and reassembles them into one message when it -// did not, that a refusal keeps the upstream's own status and words, that a dial nobody -// answered is a value the fork can move past, that the beta flags travel on their typed -// path rather than as a header, and that what the two usage-bearing events add up to is -// billed in the metrics settlement reads. +// did not, that it hands the turn's own state back for the client to carry, that a refusal +// keeps the upstream's own status and words, that a dial nobody answered is a value the fork +// can move past, that the beta flags travel on their typed path rather than as a header, and +// that what the two usage-bearing events add up to is billed in the metrics settlement reads. import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -122,7 +122,9 @@ const serveWith = async ( rememberCandidates: () => {}, rememberChatSelection: () => {}, chatPayloadFor: () => { asked += 1; return affinityPayload; }, - selectAffinity: () => {}, + // Wired where the app wires it: the carrier the edge writes is addressed to whatever the + // chain named here. + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); @@ -137,6 +139,34 @@ const collect = async (rendered: unknown): Promise => { return frames; }; +// The carrier the edge writes back is a redacted_thinking block holding this turn's own +// state. Messages has real block boundaries, so it takes content block 0 of its own: its +// start and stop are the two extra frames, and the turn's own blocks answer behind it. + +/** The turn's own frames, with the carrier block's dropped. */ +const withoutCarrier = (frames: readonly SseFrame[]): readonly SseFrame[] => { + const carrierBlocks = new Set(); + return frames.filter(frame => { + const event = JSON.parse(frame.data) as { + readonly type: string; + readonly index?: number; + readonly content_block?: { readonly type: string }; + }; + if (event.index === undefined) return true; + if (event.type === 'content_block_start' && event.content_block?.type === 'redacted_thinking') { + carrierBlocks.add(event.index); + } + return !carrierBlocks.has(event.index); + }); +}; + +/** The same events, at the block index the carrier leaves them at. */ +const behindCarrier = (events: readonly unknown[]): readonly unknown[] => + events.map(event => { + const block = event as { readonly index?: number }; + return block.index === undefined ? event : { ...block, index: block.index + 1 }; + }); + beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); affinityPayload = payload; @@ -180,14 +210,36 @@ describe('the messages chain', () => { resolves([candidate(async () => stream(turn('he', 'llo')))]); const { facts } = await serve(true); - const frames = await collect(facts['response.chat.messages.rendered']); + const frames = withoutCarrier(await collect(facts['response.chat.messages.rendered'])); expect(facts['response.http.status']).toBe(200); expect(frames.map(frame => frame.event)).toEqual([ 'message_start', 'content_block_start', 'content_block_delta', 'content_block_delta', 'content_block_stop', 'message_delta', 'message_stop', ]); - expect(frames.map(frame => frame.data)).toEqual(turn('he', 'llo').map(event => JSON.stringify(event))); + expect(frames.map(frame => frame.data)).toEqual(behindCarrier(turn('he', 'llo')).map(event => JSON.stringify(event))); + }); + + // The other half of affinity: the resolver reads client-carried state on the way down, and + // this is what the client is given to carry back. A turn that handed back nothing would + // leave the follow-up quoting it with no way to name the upstream that issued it. + it('hands the turn-s own state back on a block of its own', async () => { + resolves([candidate(async () => stream(turn('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + const frames = await collect(facts['response.chat.messages.rendered']); + const carrier = frames + .map(frame => JSON.parse(frame.data) as { readonly type: string; readonly content_block?: { readonly type: string; readonly data: string } }) + .filter(event => event.type === 'content_block_start' && event.content_block?.type === 'redacted_thinking'); + + expect(carrier).toHaveLength(1); + // It names the upstream that answered, sealed under this run's own secret — which is what + // lets the next turn be pinned to it. + expect(await gateway.affinity.codec.unwrap(carrier[0]!.content_block!.data, 'messages.redacted_thinking.data')).toMatchObject({ + kind: 'owned', + affinity: { upstreamId: 'up_a', modelId: 'claude-model' }, + }); }); // The upstream speaks SSE whatever the client asked for, so a client that did not ask to @@ -198,14 +250,14 @@ describe('the messages chain', () => { const { facts } = await serve(false); const rendered = facts['response.chat.messages.rendered'] as { readonly type: string; - readonly content: readonly { readonly text: string }[]; + readonly content: readonly { readonly type: string; readonly text?: string }[]; readonly stop_reason: string; readonly usage: { readonly input_tokens: number; readonly output_tokens: number }; }; expect(facts['response.http.status']).toBe(200); expect(rendered.type).toBe('message'); - expect(rendered.content.map(block => block.text)).toEqual(['hello']); + expect(rendered.content.filter(block => block.type !== 'redacted_thinking').map(block => block.text)).toEqual(['hello']); expect(rendered.stop_reason).toBe('end_turn'); expect(rendered.usage).toMatchObject({ input_tokens: 5, output_tokens: 7 }); }); @@ -323,7 +375,7 @@ describe('the messages chain', () => { resolves([candidate(async () => stream([...turn('hi'), textDelta(' and more')]))]); const { facts } = await serve(true); - const frames = await collect(facts['response.chat.messages.rendered']); + const frames = withoutCarrier(await collect(facts['response.chat.messages.rendered'])); expect(frames.map(frame => frame.event)).toEqual([ 'message_start', 'content_block_start', 'content_block_delta', diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index ab32a169a4..a49a265a81 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -1,9 +1,10 @@ // The Responses chain, run. Assembly is checked where every family's is; what is written // down here is what only running it can say — that the edge terminates the client's stream // on `[DONE]` however the upstream's ended, that it folds a stream into one response object -// when the client did not ask to stream, that a turn the upstream answered with one envelope -// instead of a stream is served as that envelope, that a refusal keeps the upstream's own -// status and words, and that a dial nobody answered is a value the fork can move past. +// when the client did not ask to stream, that it hands the turn's own state back for the +// client to carry, that a turn the upstream answered with one envelope instead of a stream is +// served as that envelope, that a refusal keeps the upstream's own status and words, and that +// a dial nobody answered is a value the fork can move past. import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -13,7 +14,7 @@ import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type ResponsesCompactionResult, type ResponsesResult, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import { RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type ResponsesCompactionResult, type ResponsesOutputItem, type ResponsesResult, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; import { directFetcher, type ModelCandidate, type ProviderResponsesResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -120,7 +121,9 @@ const serveWith = async (gateway: ReturnType, wantsSt rememberCandidates: () => {}, rememberChatSelection: () => {}, chatPayloadFor: () => { asked += 1; return affinityPayload; }, - selectAffinity: () => {}, + // Wired where the app wires it: the carrier the edge writes is addressed to whatever the + // chain named here. + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); @@ -135,6 +138,29 @@ const drain = async (rendered: unknown): Promise => { return frames; }; +// The carrier the edge writes back is a reasoning item holding this turn's own state. The +// message this turn answers with cannot hold it, so the carrier gets an output slot of its +// own at the head of the answer: two frames, and a slot in every snapshot that follows them. +const isCarrierItem = (item: ResponsesOutputItem): boolean => item.type === 'reasoning'; + +/** The turn's own answer, with the carrier's item taken back out. */ +const withoutCarrierItem = (response: ResponsesResult): ResponsesResult => + ({ ...response, output: response.output.filter(item => !isCarrierItem(item)) }); + +/** The turn's own frames. The two sequence numbers the carrier's frames spent stay spent — + * they are numbers the client saw. */ +const withoutCarrier = (frames: readonly SseFrame[]): readonly SseFrame[] => + frames.flatMap(frame => { + if (frame.data === '[DONE]') return [frame]; + const event = JSON.parse(frame.data) as ResponsesStreamEvent; + if ( + (event.type === 'response.output_item.added' || event.type === 'response.output_item.done') + && isCarrierItem(event.item) + ) return []; + if (!('response' in event)) return [frame]; + return [{ ...frame, data: JSON.stringify({ ...event, response: withoutCarrierItem(event.response) }) }]; + }); + beforeEach(() => { affinityPayload = payload; asked = 0; @@ -183,7 +209,7 @@ describe('the responses chain', () => { resolves([candidate(async () => stream(delta('he'), delta('llo'), completed('hello')))]); const { facts } = await serve(true); - const frames = await drain(facts['response.chat.responses.rendered']); + const frames = withoutCarrier(await drain(facts['response.chat.responses.rendered'])); expect(facts['response.http.status']).toBe(200); expect(frames.map(frame => frame.event)).toEqual([ @@ -195,11 +221,37 @@ describe('the responses chain', () => { expect(frames.map(frame => frame.data)).toEqual([ JSON.stringify(delta('he')), JSON.stringify(delta('llo')), - JSON.stringify(completed('hello')), + // The carrier's own two frames went out ahead of this one under two sequence numbers of + // their own, so the terminal event is numbered where they left it. + JSON.stringify({ ...completed('hello'), sequence_number: 4 }), '[DONE]', ]); }); + // The other half of affinity: the resolver reads client-carried state on the way down, and + // this is what the client is given to carry back. A turn that handed back nothing would + // leave the follow-up quoting it with no way to name the upstream that issued it. + it('hands the turn-s own state back on an item of its own', async () => { + resolves([candidate(async () => stream(delta('hi'), completed('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + const frames = await drain(facts['response.chat.responses.rendered']); + const answered = frames.find(frame => frame.event === 'response.completed'); + const carrier = (JSON.parse(answered!.data) as { readonly response: ResponsesResult }).response.output + .filter(item => item.type === 'reasoning'); + + expect(carrier).toHaveLength(1); + // It names the upstream that answered, sealed under this run's own secret — which is what + // lets the next turn be pinned to it. The item is ours rather than the upstream's, and it + // says so, so a turn quoting it back is not read as reasoning the upstream produced. + expect(await gateway.affinity.codec.unwrap(carrier[0]!.encrypted_content!, 'responses.reasoning.encrypted_content')).toMatchObject({ + kind: 'owned', + syntheticItem: true, + affinity: { upstreamId: 'up_a', modelId: 'responses-model' }, + }); + }); + // A stream's numbers arrive with its last chunk, so what the run hands up is a promise and // the reading is taken off the frames the client itself drove. it('bills what the terminal event reported, once the frames have run out', async () => { @@ -241,7 +293,7 @@ describe('the responses chain', () => { resolves([candidate(async () => stream(delta('hi'), completed('hi'), delta(' and more')))]); const { facts } = await serve(true); - const frames = await drain(facts['response.chat.responses.rendered']); + const frames = withoutCarrier(await drain(facts['response.chat.responses.rendered'])); expect(frames.map(frame => frame.event)).toEqual([ 'response.output_text.delta', @@ -269,7 +321,7 @@ describe('the responses chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(200); - expect(facts['response.chat.responses.rendered']).toMatchObject({ + expect(withoutCarrierItem(facts['response.chat.responses.rendered'] as unknown as ResponsesResult)).toMatchObject({ id: 'resp_1', status: 'completed', output: [{ content: [{ type: 'output_text', text: 'hello' }] }], From ae8bbd48c02ab23815f19d83e4384732f51a4d08 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 07:12:19 +0800 Subject: [PATCH 033/156] style(chat): write the services generic as written, not as patched Two edges carried the ChatServices type argument on a line of its own after a dangling comma, which is how a script left them rather than how anyone would write them. --- packages/gateway/src/data-plane/chat/messages/pipeline.ts | 3 +-- packages/gateway/src/data-plane/chat/responses/pipeline.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index 012122b528..e934fc47d4 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -81,8 +81,7 @@ const emitMessages = defineStage< M<'ingress.chat.messages.wantsStream'>, M<'ingress.chat.messages.wantsStream'>, M<'ingress.chat.messages.wantsStream' | 'response.chat.messages' | 'response.http.headers'>, - M<'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers'> - , + M<'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ name: 'emitMessages', diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index a2664b5134..aa6c9534a3 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -102,8 +102,7 @@ const emitResponses = defineStage< R<'ingress.chat.responses.wantsStream'>, R<'ingress.chat.responses.wantsStream'>, R<'ingress.chat.responses.wantsStream' | 'response.chat.responses' | 'response.http.headers'>, - R<'response.chat.responses.rendered' | 'response.http.status' | 'response.http.headers'> - , + R<'response.chat.responses.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ name: 'emitResponses', From e018952a1e216d36d896044e561da102eafea7f3 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 07:53:50 +0800 Subject: [PATCH 034/156] refactor(chat): put each family's wire under a fork its last stage chooses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every chat chain ended in one stage that dialled one protocol, so a candidate reachable only over another wire had nowhere to go. This replaces that ending with `dialChatWire`, an `into` stage that reads which wire the candidate is reachable on and hands into the chain for it. Only a last stage may name a target, and failover is an ordinary middle stage — so re-running the suffix re-runs this one and the next candidate re-picks, with no mechanism of its own. Each protocol's own wire becomes a chain a source can hand into rather than a stage only its own family can reach: `chatCompletionsWire`, `messagesWire` and `responsesWire`. Where a wire hands its reading is the *source* family's key, so each is built with it rather than naming one of its own. Which rules sit in a wire rather than above the fork is the same question the interceptor form answered with `ctx.targetApi !== `. The four rules that carried that guard — the three role rewrites and the Responses cache-bucket fold — speak about one protocol's endpoint, so they move into that protocol's wire. Position says what the guard said, and `compose` refuses to place a rule that reads a request key where a handoff has consumed it. Gemini's fused ending splits into the same shape: a handoff into Chat Completions plus that protocol's wire, with the turn's own end read where the protocol the client speaks is, in `requireGeminiTerminal`. Anthropic's beta flags now travel only when the client spoke Messages. They have a typed path no header allowlist admits, and a turn that arrived through a translation asked for nothing on that wire. No wire changes hands yet: every family still reaches only the protocol it reached before. --- .../data-plane/chat/gemini-pipeline_test.ts | 17 +- .../chat/chat-completions/pipeline.ts | 80 +++-- .../src/data-plane/chat/gemini/pipeline.ts | 287 ++++++------------ .../gateway/src/data-plane/chat/handoff.ts | 96 +++++- .../src/data-plane/chat/messages/pipeline.ts | 83 +++-- .../src/data-plane/chat/responses/pipeline.ts | 93 ++++-- 6 files changed, 371 insertions(+), 285 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index 3b75f52d16..fe57895ba5 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -187,11 +187,15 @@ const affinityCarriers = (events: readonly GeminiResult[]): readonly string[] => describe('the gemini pipeline', () => { // The whole entry contract, and it does not mention `request.chat.gemini`: the turn the - // ending translates is not the one the caller handed in but the one materializeAttempt put + // wire translates is not the one the caller handed in but the one materializeAttempt put // into the record below the fork, which is where a per-candidate payload can exist at all. + // The two `ingress.*` keys are there because the fork declares what the wire under it + // reads — a wire is built against a candidate, so assembly cannot ask one itself. it('assembles, and asks its caller only for what the descending stages need', () => { expect([...geminiServePipeline(payload).entryNeeds].sort()).toEqual([ 'ingress.chat.gemini.wantsStream', + 'ingress.chat.sourceProtocol', + 'ingress.http.headers', 'serve.model', ]); }); @@ -226,7 +230,7 @@ describe('the gemini pipeline', () => { // upstream that issued them — so the turn that is translated is the one affinity rewrote // for this candidate, and the candidate that answered is marked for the next turn. That // turn enters the record below the fork because everything between there and the dial - // rewrites the request as a fact, and one stage owns the reading: an ending that asked the + // rewrites the request as a fact, and one stage owns the reading: a wire that asked the // resolver again would translate a turn no stage in the chain had touched. it('translates what the record holds, and marks the candidate that answered', async () => { let sent: Record | undefined; @@ -240,9 +244,12 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts()); await collect(facts['response.chat.gemini.rendered']); - // The exit contract names what the run answers with, and the turn the chain translated is - // not that — but the record carries every key in flight, so it is still there to read. - expect((facts as Record)['request.chat.gemini']).toBe(affinityPayload); + // The handoff consumed `request.chat.gemini` and provided the wire's own key, so what the + // record still carries below the fork is the translated turn and not the Gemini one — the + // whole of what a translation is, said in the fact space. + expect((facts as Record)['request.chat.gemini']).toBeUndefined(); + expect((facts as Record)['request.chat.chatCompletions']) + .toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); expect(asked).toBe(1); expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); expect(selected).toEqual([dialled]); diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index db62a8610c..cbfa260ab0 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -5,12 +5,19 @@ // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // the interceptors the shared array, each one a stage -// callChatCompletionsUpstream the ending: dials this candidate's wire +// dialChatWire the ending: picks this candidate's wire and hands into it // // The wire a candidate is dialled on is chosen per candidate rather than at assembly, and -// because failover re-runs the whole suffix the next candidate re-picks its own. Only the -// native wire is here; the translated ones hand up the same key, which is what will make -// them interchangeable with it. +// because failover re-runs the whole suffix the next candidate re-picks its own. Only its own +// wire is built here; the translated ones will hand up `response.chat.chatCompletions` too, +// which is what will make them interchangeable with it. +// +// What sits *in* a wire rather than above the fork is the other half of the arrangement. A +// rule that speaks about an upstream's Chat Completions endpoint — the role rewrite does, +// which is why its interceptor form stood down on `ctx.targetApi !== 'chat-completions'` — +// belongs to the wire and not to the source chain, so a turn that leaves for another +// protocol never carries it. Said by position rather than by a guard: a stage below the fork +// runs only when this is the wire the fork chose. import { wrapChatCompletionsAffinityEgress } from './affinity/egress.ts'; import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; @@ -25,6 +32,7 @@ import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; +import { dialChatWire, type ChatWire } from '../handoff.ts'; import { applyRoleCompatibilityToChatCompletions, disableReasoningOnForcedToolChoiceForChatCompletions, @@ -35,7 +43,7 @@ import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; import { CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, chatCompletionsErrorPayloadMessage, @@ -164,21 +172,25 @@ const renderSSE = ( }); /** - * The native wire. It dials Chat Completions and provides the answer at this family's own - * response key — which is what will make it interchangeable with a translated chain: both - * hand up `response.chat.chatCompletions`, and the stage above cannot tell which ran. + * The wire. It dials Chat Completions and provides the answer at whichever family's response + * key the chain above it reads — which is what makes it interchangeable with a translated + * chain: both hand up `response.chat.chatCompletions`, and the stage above cannot tell which + * ran. + * + * `streamedUsage` is the one key it is told. A wire hands up one reading whatever protocol it + * spoke, and where that reading lands is the *source* family's own key rather than this + * protocol's — so it is built with it rather than naming one of its own. */ -const callChatCompletionsUpstream = defineStage< +const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< C<'request.chat.chatCompletions' | 'route.attempt' | 'ingress.http.headers'>, - C<'response.chat.chatCompletions' | 'response.chat.chatCompletions.streamedUsage' - | 'response.usage.billable' | 'response.http.headers'>, + C<'response.chat.chatCompletions' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ name: 'callChatCompletionsUpstream', return: { provides: [ 'response.chat.chatCompletions', - 'response.chat.chatCompletions.streamedUsage', + streamedUsage, 'response.usage.billable', 'response.http.headers', ], @@ -190,9 +202,10 @@ const callChatCompletionsUpstream = defineStage< use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); // What the record holds by now: the payload affinity materialized for this candidate, - // as every stage between the fork and here has rewritten it. The id the client addressed - // does not travel — the provider re-stamps whatever it resolved upstream — and an alias' - // own rules apply to the body that is sent. + // as every stage between the fork and here has rewritten it — or, on a translated wire, + // what the handoff put here. The id the client addressed does not travel — the provider + // re-stamps whatever it resolved upstream — and an alias' own rules apply to the body + // that is sent. const payload = { ...facts['request.chat.chatCompletions'], model: candidate.model.id }; if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(payload, candidate.rules); const { model: _addressed, ...body } = payload; @@ -215,7 +228,7 @@ const callChatCompletionsUpstream = defineStage< return move({ ...facts, 'response.chat.chatCompletions': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.chat.chatCompletions.streamedUsage': null, + [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], }); @@ -238,7 +251,7 @@ const callChatCompletionsUpstream = defineStage< message: text, ...(parsed === undefined ? {} : { body: parsed }), }, - 'response.chat.chatCompletions.streamedUsage': null, + [streamedUsage]: null, 'response.usage.billable': called, 'response.http.headers': [...result.response.headers], }); @@ -251,13 +264,35 @@ const callChatCompletionsUpstream = defineStage< return move({ ...facts, 'response.chat.chatCompletions': { kind: 'stream' as const, frames: metered.frames }, - 'response.chat.chatCompletions.streamedUsage': metered.outcome, + [streamedUsage]: metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); }, }); +/** + * The Chat Completions wire, as the chain that dials it. + * + * Every source protocol that reaches an upstream over this endpoint runs this, whether the + * client spoke Chat Completions or a handoff arrived here — which is what makes a rule that + * speaks about *this* wire belong here. The role rewrite states what an upstream's Chat + * Completions endpoint accepts, so it applies to whatever body this wire actually sends and + * to nothing that leaves for another protocol. + */ +export const chatCompletionsWire = (streamedUsage: string): readonly Stage[] => [ + applyRoleCompatibilityToChatCompletions, + callChatCompletionsUpstream(streamedUsage), +]; + +/** This family's own reading, which every wire under it hands up. */ +const STREAMED_USAGE = 'response.chat.chatCompletions.streamedUsage'; + +/** The wires `/v1/chat/completions` can be served on. Only its own is built; the two + * translated ones are each a handoff and then that protocol's own wire, and until they exist + * a candidate the picker admitted on one of them is dialled here anyway. */ +const chatCompletionsWireFor = (): ChatWire => compose('chatCompletionsNative', chatCompletionsWire(STREAMED_USAGE)); + /** Reads the upstream's own usage off its own events as they pass, so the reading costs one * pass and the client's stream is what drives it. Only a report carrying real counts * replaces the running figure, so a trailing empty usage frame cannot wipe a good one. */ @@ -368,8 +403,13 @@ export const chatCompletionsServePipeline = (payload: ChatCompletionsPayload): P owns: [], }), materializeAttempt('request.chat.chatCompletions'), - applyRoleCompatibilityToChatCompletions, disableReasoningOnForcedToolChoiceForChatCompletions, stripPromptCacheKeyForChatCompletions, - callChatCompletionsUpstream, + dialChatWire({ + source: 'request.chat.chatCompletions', + needs: ['request.chat.chatCompletions', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.chatCompletions', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => chatCompletionsTarget.pick(endpoints), + wire: chatCompletionsWireFor, + }), ]); diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index ab43eeb4a1..05e261e1d2 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -5,7 +5,8 @@ // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record -// callGeminiViaChatCompletions the ending: translates, dials, and meters what came back +// the strippers, then requireGeminiTerminal +// dialChatWire the ending: picks this candidate's wire and hands into it // // Gemini is source-only. Nothing translates *into* it, so it contributes one pipeline rather // than two — a source role and no target role, and no other family's chain ever hands up @@ -13,22 +14,28 @@ // // It is also the one family with no wire of its own: the provider surface has no `callGemini`, // so every candidate is reached through a translation and the wire is whichever of the three -// translatable ones the picker finds on it. Only the first, Chat Completions, is built here. +// translatable ones the picker finds on it. Only the first, Chat Completions, is built here — +// a handoff into that protocol, followed by that protocol's own wire. +// +// Because every wire is translated, the reading is taken on the dialect the upstream actually +// spoke: the target protocol's own wire meters its own frames on their way into the +// translation, so a figure the upstream stated is billed as stated. Where the *turn* ends is +// a different question and is stated on the Gemini frames the client is handed, which is what +// `requireGeminiTerminal` is for — a Chat Completions stream that closed cleanly without ever +// reporting a finish reason translates into candidates that never finish. // // What is not built, stated rather than implied: -// - the Messages and the Responses wires. Each is its own ending stage — a different -// translate pair, a different provider method, and a different event dialect to meter on -// — and until they exist a candidate that `canServe` admitted on one of those two is -// dialled on Chat Completions anyway, which is the same interim state the Chat -// Completions chain carries for its own two translated wires. +// - the Messages and the Responses wires. Each is a handoff of its own into that protocol's +// wire, and until they exist a candidate the picker admitted on one of those two is +// dialled on Chat Completions anyway, which is the same interim state the other three +// chains carry for their own translated wires. // - `:countTokens`. It is a second entry against this protocol rather than another wire of // this one: no stream, no billable turn, and a `{ totalTokens }` envelope of its own. -// - the Gemini interceptors and the affinity egress. The four `interceptors/` entries and -// the thought-signature rewrite on the way out are still only in the interceptor form. +// - the affinity egress' thought-signature rewrite is here, but the Gemini interceptors are +// only partly stages: the four `interceptors/` entries are, and nothing else is. // - the Google-RPC envelope `respond.ts` wrote into a client's own stream for a turn that -// ended without a terminal event. The chain detects one — the ending stops reading at -// the terminal frame and throws when the frames run out before one — but it leaves the -// run as a throw rather than as that 502 envelope. +// ended without a terminal event. The chain detects one and leaves the run as a throw +// rather than as that 502 envelope. // // One deliberate difference from `respond.ts`, shared with every other family on the // pipeline: an upstream that refused is answered in its own words. A Chat Completions wire @@ -39,32 +46,26 @@ import { analyzeGeminiAffinity } from './affinity/ingress.ts'; import { renderGeminiError } from './errors.ts'; -import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; -import { billableUsageFromChatCompletionsEvent } from '../chat-completions/usage.ts'; +import { chatCompletionsWire } from '../chat-completions/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; +import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { stripSafetySettingsFromGemini, stripUnsupportedPartFieldsFromGemini, stripUnsupportedToolsFromGemini, suppressThoughtPartsFromGemini, } from '../interceptors.ts'; -import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; -import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { compose, defineStage, move, transform, type Pipeline } from '@floway-dev/pipeline'; +import { renderProtocolError, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectGeminiProtocolEventsToResult, geminiProtocolFrameToSSEFrame, @@ -73,7 +74,7 @@ import { type GeminiPayload, type GeminiStreamEvent, } from '@floway-dev/protocols/gemini'; -import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; import { translateGeminiViaChatCompletions } from '@floway-dev/translate'; /** `:generateContent` has no wire of its own, so the whole preference list is translated: @@ -186,183 +187,80 @@ const renderSSE = (frames: AsyncIterable>): Asy }); /** - * The Chat Completions wire. It translates the turn, dials, and hands the answer up at this - * family's own response key — which is what makes it interchangeable with the two wires that - * are not built yet: all three would provide `response.chat.gemini`, and the stage above - * cannot tell which ran. + * Says where a Gemini turn ends, on the frames the client is handed rather than on the wire + * below them. + * + * The two are not the same statement. A Chat Completions stream that closed cleanly without + * ever reporting a finish reason ends its own dialect properly and translates into candidates + * that never finish, and serving those would report a truncated answer as a whole one. So the + * turn's end is read here, where the protocol the client speaks is, and the wire's end is read + * on the wire. * - * The reading is taken on the dialect the upstream actually spoke. Usage is metered off the - * Chat Completions frames on their way into the translation rather than off the Gemini frames - * that come out of it, so a figure the upstream stated is billed as stated. + * It sits below the thought suppressor and above the fork: what it truncates is what the rules + * above it then work on, which is the order the fused ending had. */ -const callGeminiViaChatCompletions = defineStage< - C<'request.chat.gemini' | 'route.attempt' | 'ingress.http.headers'>, - C<'response.chat.gemini' | 'response.chat.gemini.streamedUsage' - | 'response.usage.billable' | 'response.http.headers'>, - ChatServices +const requireGeminiTerminal = defineStage< + Record, + Record, + C<'response.chat.gemini'>, + C<'response.chat.gemini'> >({ - name: 'callGeminiViaChatCompletions', - return: { - provides: [ - 'response.chat.gemini', - 'response.chat.gemini.streamedUsage', - 'response.usage.billable', - 'response.http.headers', - ], + name: 'requireGeminiTerminal', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: ['response.chat.gemini'], consumes: [], provides: ['response.chat.gemini'] }, }, - execute: async (facts, use) => { - const candidate = use.resolveAttempt(facts['route.attempt']); - // Attribution is set before the dial — before the translation too, which can reject the - // client's own input — so an attempt that never completes still names the candidate it was - // made against rather than the one tried before it. - use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - - // The trip owns both directions: it builds the payload that goes out and hands back the - // closure that turns the upstream's frames into Gemini's. What it translates is what the - // record holds by now — the turn affinity materialized for this candidate, as every stage - // between the fork and here has rewritten it, with client-carried state rewritten for the - // upstream that will see it. The id the client addressed does not travel: Gemini carries - // it in the URL, the trip stamps the candidate's own, and the wire is handed the body - // without it. An alias' own rules apply to the translated body, which is the only shape - // the wire will see. - const trip = await translateGeminiViaChatCompletions(facts['request.chat.gemini'], { model: candidate.model.id }); - if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(trip.target, candidate.rules); - const { model: _addressed, ...body } = trip.target; - - let result; - try { - result = await candidate.provider.instance.callChatCompletions( - providerModelOf(candidate), - body, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.chat.gemini': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.chat.gemini.streamedUsage': null, - 'response.usage.billable': [], - 'response.http.headers': [], - }); - } - - const identity = telemetryModelIdentity(candidate, result.modelKey); - // An upstream that was called and reported nothing, which is a different statement from - // reporting zero. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; - - if (!result.ok) { - const text = await result.response.text(); - use.log.warn('upstream refused', { status: result.response.status }); - let parsed: unknown; - try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } - return move({ + execute: transform< + Record, + Record, + C<'response.chat.gemini'>, + C<'response.chat.gemini'> + >(() => ({ + response: facts => { + const answer = facts['response.chat.gemini']; + // A refusal and a collected body never opened a stream, so there is no end to find. + if (isFailure(answer) || answer.kind !== 'stream') return facts; + const frames = answer.frames as AsyncIterable>; + return { ...facts, - 'response.chat.gemini': { - status: result.response.status, - message: text, - ...(parsed === undefined ? {} : { body: parsed }), - }, - 'response.chat.gemini.streamedUsage': null, - 'response.usage.billable': called, - 'response.http.headers': [...result.response.headers], - }); - } - - // This candidate answered, so it is the one a follow-up turn carrying our own state must - // come back to. - use.selectAffinity(candidate); - // Where this turn ends is stated on the Gemini frames the client is handed, and what it - // billed is read on the Chat Completions wire below them. - const ending = { sawTerminal: false }; - const metered = meterChatCompletions(result.events, identity, use.gateway.attempt, ending); - return move({ - ...facts, - 'response.chat.gemini': { kind: 'stream' as const, frames: geminiFramesUntilTerminal(trip.events(metered.frames), ending) }, - 'response.chat.gemini.streamedUsage': metered.outcome, - 'response.usage.billable': called, - 'response.http.headers': [...(result.headers ?? new Headers())], - }); - }, + 'response.chat.gemini': move({ + kind: 'stream' as const, + frames: { [Symbol.asyncIterator]: () => framesUntilTerminal(frames) }, + }), + }; + }, + })), }); -/** Reads the upstream's own usage off its own events as they pass, so the reading costs one - * pass and the client's stream is what drives it. Only a report carrying real counts - * replaces the running figure, so a trailing empty usage frame cannot wipe a good one. */ -const meterChatCompletions = ( - source: AsyncIterable>, - identity: TelemetryModelIdentity, - attempt: { firstOutputTokenAt: number | null }, - ending: { sawTerminal: boolean }, -): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { - let settle!: (outcome: StreamOutcome) => void; - const outcome = new Promise(resolve => { settle = resolve; }); - const generator = (async function* () { - let reported: BillableUsage | undefined; - try { - for await (const frame of source) { - // Time to first token is measured where the token is, which is the only place that - // knows a frame carries generated content rather than the envelope around it — and - // it is read on the dialect the upstream spoke, like the usage beside it. - if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'chat-completions')) { - attempt.firstOutputTokenAt = performance.now(); - } - if (frame.type === 'event') { - const usage = billableUsageFromChatCompletionsEvent(frame.event); - if (usage !== null) reported = usage; - } - yield frame; - } - } finally { - // Reached however the frames ended — the terminal chunk, a client that stopped - // reading, or a broken upstream — because tokens the upstream already metered are - // billable whatever happened to the downstream half. - settle({ billable: [billedEntity(reported, identity)], failed: !ending.sawTerminal }); - } - })(); - return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; +/** Stops at the frame that ends the turn — there is nothing further to read, and returning + * here closes the translation and the wire under it — and fails a stream that ran out before + * one arrived. */ +const framesUntilTerminal = async function* ( + frames: AsyncIterable>, +): AsyncGenerator> { + for await (const frame of frames) { + yield frame; + if (frame.type === 'done' || isGeminiTerminalEvent(frame.event)) return; + } + throw new Error(GEMINI_MISSING_TERMINAL_MESSAGE); }; -/** Where a Gemini turn ends, stated on the frames the client is handed rather than on the - * wire below: a Chat Completions stream that closed cleanly without ever reporting a finish - * reason translates into candidates that never finish, and serving those would report a - * truncated answer as a whole one. */ -const geminiFramesUntilTerminal = ( - frames: AsyncIterable>, - ending: { sawTerminal: boolean }, -): AsyncIterable> => ({ - [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) { - yield frame; - // The turn is over, so there is nothing further to read; returning here closes the - // translation and the wire under it. The end is stated here and the usage is read on - // the wire below, so one flag carries what the settlement needs from both. - if (frame.type === 'done' || isGeminiTerminalEvent(frame.event)) { - ending.sawTerminal = true; - return; - } - } - throw new Error(GEMINI_MISSING_TERMINAL_MESSAGE); - })(), -}); +/** This family's own reading, which every wire under it hands up. */ +const STREAMED_USAGE = 'response.chat.gemini.streamedUsage'; -/** What one attempt is billable for. An upstream that reported nothing leaves no quantities - * at all, which is a different statement from reporting zero — and a rate can depend on the - * service tier and on how much input there was, so both travel as pricing facts rather than - * being folded into the quantities. */ -const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { - const tokens = tokenUsageFromBillableUsage(usage); - if (tokens === null) return { identity, quantities: {} }; - const measurement = tokenUsageMeasurement(tokens); - return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; -}; +/** The wires `:generateContent` can be served on, and every one of them is a translated one: + * the provider surface has no Gemini call, so a handoff is how every candidate is reached. + * Only the first preference is built; until the other two exist a candidate the picker + * admitted on one of them is dialled here anyway. */ +const geminiWireFor = (_target: ChatTargetApi, candidate: ModelCandidate): ChatWire => + compose('geminiViaChatCompletions', [ + handOff({ + from: { request: 'request.chat.gemini', response: 'response.chat.gemini' }, + to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + trip: async payload => await translateGeminiViaChatCompletions(payload, { model: candidate.model.id }), + }), + ...chatCompletionsWire(STREAMED_USAGE), + ]); /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is @@ -407,5 +305,12 @@ export const geminiServePipeline = (payload: GeminiPayload): Pipeline geminiTarget.pick(endpoints), + wire: geminiWireFor, + }), ]); diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index 7d1ae7daa6..8e420677f2 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -1,9 +1,9 @@ -// A protocol handoff, as a stage. +// A protocol handoff, as a stage — and the stage that chooses one. // -// This is the whole of what a translation is in the fact space: it **consumes** the source -// protocol's request key and **provides** the target's, and coming back it consumes the -// target's response key and provides the source's. Two consequences follow from saying it -// that way rather than carrying a `targetApi` field. +// A handoff is the whole of what a translation is in the fact space: it **consumes** the +// source protocol's request key and **provides** the target's, and coming back it consumes +// the target's response key and provides the source's. Two consequences follow from saying +// it that way rather than carrying a `targetApi` field. // // The source key is gone below this stage, so a stage that needs it cannot be placed there // and `compose` says so at assembly. That is a check on one chain's arrangement and not a @@ -12,17 +12,25 @@ // // And nothing below knows a translation happened. The target chain sees its own protocol's // keys and only those, so the same chain serves a native request and a translated one. +// +// `dialChatWire` is the other half: the last stage of a source chain, which reads which wire +// this candidate is reachable on and hands into the chain for it. Only a last stage may name +// a target, and failover is an ordinary middle stage — so re-running the suffix re-runs this +// one, and the next candidate re-picks with no mechanism of its own. import type { ChatAnswer, ChatFacts } from './facts.ts'; +import type { ChatServices } from './stages.ts'; +import type { AttemptSelector } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; -import { defineStage, move } from '@floway-dev/pipeline'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { defineStage, move, type Pipeline, type Use } from '@floway-dev/pipeline'; +import type { ModelEndpoints, ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; import type { TranslateTripResult } from '@floway-dev/translate'; type RequestKey = Extract; type ResponseKey = Extract; -export interface Handoff { +export interface Handoff { /** The two request keys, and the two response keys they answer at. Naming all four is what * lets the declaration be written; a pair is not derivable from one half. */ readonly from: { readonly request: Source; readonly response: ResponseKey }; @@ -30,8 +38,13 @@ export interface Handoff { /** The pair function, already closed over whatever context it reads. It returns the target * payload together with the closure that maps the target's frames back — trip-scoped * state (synthetic ids, custom-tool name sets) lives in that closure and the source chain - * never sees it. */ - readonly trip: (source: ChatFacts[Source]) => Promise>; + * never sees it. + * + * Only the *target's* event type is named. It is what the pair's mapping closure is handed, + * so leaving it open would make the frames coming off the wire unassignable to it; what the + * closure produces goes straight to the source's response key, which no declaration here + * looks inside. */ + readonly trip: (source: ChatFacts[Source]) => Promise>; } /** @@ -41,8 +54,8 @@ export interface Handoff { * context-window error becoming the Anthropic `prompt is too long:` shape that Claude Code * reads for auto-compaction. A pair that declares no rewrite hands the refusal up unchanged. */ -export const handOff = ( - handoff: Handoff, +export const handOff = ( + handoff: Handoff, ) => defineStage< { [K in Source]: ChatFacts[Source] }, { [K in Target]: ChatFacts[Target] }, @@ -106,8 +119,65 @@ export const handOff = ( ...up, [handoff.from.response]: move({ kind: 'stream', - frames: trip.events(answer.frames as AsyncIterable>), + frames: trip.events(answer.frames as AsyncIterable>), }), } as never; }, }); + +/** One wire, as the chain that dials it. The keys are erased at this seam for the same + * reason they are erased on `Stage`: what a wire reads and hands up is checked by assembly + * and by the runner against declarations, and a family's slice type is nobody's business + * in between. */ +export type ChatWire = Pipeline, Record>; + +/** How a source protocol reaches an upstream. */ +export interface ChatWiring { + /** The request key of the protocol the client spoke. It names the stage, so a dump says + * which chain forked without the name having to be written twice. */ + readonly source: RequestKey; + /** What the chain below this stage reads. A wire is built against the candidate it will + * dial, so assembly cannot ask one what it needs — the family says it here, and that is + * what puts those keys in the serve pipeline's entry contract. */ + readonly needs: readonly string[]; + /** What comes back through here, whichever wire ran. Every wire hands up this family's own + * keys, which is what makes them interchangeable. */ + readonly provides: readonly string[]; + /** Which wire this candidate is reachable on. Total by contract: serve narrowed the + * candidates with the same picker's `canServe`, so a candidate that reached here has one + * of the endpoints the preference list names. */ + readonly pick: (endpoints: ModelEndpoints) => ChatTargetApi; + /** The chain for one wire, built against the candidate that will be dialled — which is what + * lets a translation close over the model it is translating for. */ + readonly wire: (target: ChatTargetApi, candidate: ModelCandidate, use: Use) => ChatWire; +} + +/** + * Picks this candidate's wire and hands into the chain for it. + * + * It is last, which is what earns it the right to name a target at all, and it holds no + * state across candidates: what re-decides for the next candidate is failover re-running the + * suffix, which re-runs this stage. + */ +export const dialChatWire = (wiring: ChatWiring) => defineStage< + { 'route.attempt': AttemptSelector } & Record, + Record, + Record, + Record, + ChatServices +>({ + name: `dialChatWire:${wiring.source}`, + into: { + request: { needs: ['route.attempt', ...wiring.needs], consumes: [], provides: [] }, + // Nothing is read on the way back — a wire hands up this family's own keys and they ride + // through — but this is where they enter the chain, so this is the stage that provides + // them and the runner checks that the wire delivered. + response: { needs: [], consumes: [], provides: wiring.provides }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const target = wiring.pick(candidate.model.endpoints); + use.log.debug('dialling', { upstream: facts['route.attempt'].upstreamId, wire: target }); + return await next(facts, wiring.wire(target, candidate, use)); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index e934fc47d4..6522db65b4 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -5,14 +5,14 @@ // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record -// callMessagesUpstream the ending: dials this candidate's wire +// dialChatWire the ending: picks this candidate's wire and hands into it // -// Only the native wire is here. The translated ones — Messages via Responses and Messages -// via Chat Completions — hand up the same `response.chat.messages` key, which is what will -// make them interchangeable with this one, and each is a step of its own. So is -// `/v1/messages/count_tokens`: a second operation over this protocol rather than another -// wire under this pipeline. This family's own interceptors are not stages yet either, so -// the array between the materialized payload and the ending is empty rather than short. +// Only its own wire is built. The translated ones — Messages via Responses and Messages via +// Chat Completions — will hand up `response.chat.messages` too, which is what will make them +// interchangeable with it, and each is a step of its own. So is +// `/v1/messages/count_tokens`: a second operation over this protocol rather than another wire +// under this pipeline. This family's own interceptors are not all stages yet either, so the +// array between the materialized payload and the fork is short rather than complete. import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { renderMessagesError } from './errors.ts'; @@ -27,6 +27,7 @@ import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; +import { dialChatWire, type ChatWire } from '../handoff.ts'; import { applyRoleCompatibilityToMessages, disableReasoningOnForcedToolChoiceForMessages, @@ -39,7 +40,7 @@ import { wrapMessagesAffinityEgress } from './affinity/egress.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { isClaudeCodeProbe, probeFrames } from './interceptors/answer-claude-code-probe.ts'; -import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; import { renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { collectMessagesProtocolEventsToResult, @@ -237,17 +238,16 @@ const answerClaudeCodeProbe = defineStage< }, }); -const callMessagesUpstream = defineStage< - M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers'>, - M<'response.chat.messages' | 'response.chat.messages.streamedUsage' - | 'response.usage.billable' | 'response.http.headers'>, +const callMessagesUpstream = (streamedUsage: string) => defineStage< + M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, + M<'response.chat.messages' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ name: 'callMessagesUpstream', return: { provides: [ 'response.chat.messages', - 'response.chat.messages.streamedUsage', + streamedUsage, 'response.usage.billable', 'response.http.headers', ], @@ -259,11 +259,11 @@ const callMessagesUpstream = defineStage< use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); // What the record holds by now: the payload affinity materialized for this candidate, as - // every stage between the fork and here has rewritten it. A thinking signature was - // rewritten for the upstream that will see it, and one that no upstream but the issuer - // can read was dropped rather than sent on. The id the client addressed does not travel — - // the provider re-stamps whatever it resolved upstream — and an alias' own rules apply to - // the body that is sent. + // every stage between the fork and here has rewritten it — or, on a translated wire, what + // the handoff put here. A thinking signature was rewritten for the upstream that will see + // it, and one that no upstream but the issuer can read was dropped rather than sent on. + // The id the client addressed does not travel — the provider re-stamps whatever it + // resolved upstream — and an alias' own rules apply to the body that is sent. const payload = { ...facts['request.chat.messages'], model: candidate.model.id }; if (candidate.rules !== undefined) applyRulesToUpstreamMessages(payload, candidate.rules); const { model: _addressed, ...body } = payload; @@ -271,10 +271,14 @@ const callMessagesUpstream = defineStage< // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump // shows what was there to filter. Anthropic's beta flags are the exception — they have a - // typed path of their own so no header allowlist can admit them and no other source - // protocol can leak them in, which is why the field is read off and then dropped. + // typed path of their own so no header allowlist can admit them, and they are the + // client's own only when the client spoke this protocol: a turn that arrived here through + // a translation asked for nothing on this wire, so the field is read off and then dropped + // whichever protocol sent it. const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); - const anthropicBeta = parseAnthropicBetaHeader(headers.get('anthropic-beta')); + const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'messages' + ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) + : []; headers.delete('anthropic-beta'); let result; @@ -292,7 +296,7 @@ const callMessagesUpstream = defineStage< return move({ ...facts, 'response.chat.messages': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.chat.messages.streamedUsage': null, + [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], }); @@ -315,7 +319,7 @@ const callMessagesUpstream = defineStage< message: text, ...(parsed === undefined ? {} : { body: parsed }), }, - 'response.chat.messages.streamedUsage': null, + [streamedUsage]: null, 'response.usage.billable': called, 'response.http.headers': [...result.response.headers], }); @@ -328,13 +332,35 @@ const callMessagesUpstream = defineStage< return move({ ...facts, 'response.chat.messages': { kind: 'stream' as const, frames: metered.frames }, - 'response.chat.messages.streamedUsage': metered.outcome, + [streamedUsage]: metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); }, }); +/** + * The Messages wire, as the chain that dials it. + * + * Every source protocol that reaches an upstream over this endpoint runs this, whether the + * client spoke Messages or a handoff arrived here — which is what makes a rule that speaks + * about *this* wire belong here. The system-role rewrite states what an upstream's Messages + * endpoint accepts, so it applies to whatever body this wire actually sends and to nothing + * that leaves for another protocol. + */ +export const messagesWire = (streamedUsage: string): readonly Stage[] => [ + applyRoleCompatibilityToMessages, + callMessagesUpstream(streamedUsage), +]; + +/** This family's own reading, which every wire under it hands up. */ +const STREAMED_USAGE = 'response.chat.messages.streamedUsage'; + +/** The wires `/v1/messages` can be served on. Only its own is built; the two translated ones + * are each a handoff and then that protocol's own wire, and until they exist a candidate the + * picker admitted on one of them is dialled here anyway. */ +const messagesWireFor = (): ChatWire => compose('messagesNative', messagesWire(STREAMED_USAGE)); + /** Reads the upstream's own usage off its own events as they pass, so the reading costs one * pass and the client's stream is what drives it. Anthropic states input accounting on * `message_start` and output accounting on `message_delta`, so the reader that merges the @@ -446,6 +472,11 @@ export const messagesServePipeline = (payload: MessagesPayload): Pipeline messagesTarget.pick(endpoints), + wire: messagesWireFor, + }), ]); diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index aa6c9534a3..cc36879c21 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -5,11 +5,15 @@ // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record -// callResponsesUpstream the ending: dials this candidate's wire +// dialChatWire the ending: picks this candidate's wire and hands into it // -// One wire and one transport: `/v1/responses` over HTTP, dialled on a candidate's own -// Responses endpoint. Everything else this protocol owns is a step of its own, and is stated -// here rather than implied by its absence: +// Only its own wire is built. What sits *in* a wire rather than above the fork is a rule that +// speaks about that wire: the role rewrite and the cache-bucket fold both do, which is why +// their interceptor forms stood down on `ctx.targetApi !== 'responses'`, and position says it +// here instead. +// +// One transport, and everything else this protocol owns is a step of its own, stated here +// rather than implied by its absence: // // - the WebSocket transport. A second entry against this protocol — its own framing, its // own lifecycle — and `websocket.ts` still owns it. @@ -17,12 +21,12 @@ // wire under this pipeline, so it stays on `responsesServe.compact`. The dial here asks // for `generate`; what the ending answers with is the branch the provider says it ran, // which is why the envelope a compaction is arrives somewhere rather than nowhere. -// - the translated wires, Responses via Messages and via Chat Completions. Each hands up -// the same `response.chat.responses` key, which is what will make them interchangeable -// with this one. -// - this family's interceptors, the server-tool shims among them. Still only in the -// interceptor form, so the array between the materialized payload and the ending is -// empty rather than short. +// - the translated wires, Responses via Messages and via Chat Completions. Each will hand +// up `response.chat.responses` too, which is what will make them interchangeable with +// this one. +// - this family's remaining interceptors, the server-tool shims among them. Still only in +// the interceptor form, so the array between the materialized payload and the fork is +// short rather than complete. // - the stored-items membrane: `previous_response_id` expansion, item hydration, and the // snapshot the next turn reads. It wraps the chain rather than sitting in it, and // `serve-prep.ts` and `client-output.ts` still hold it. @@ -46,6 +50,7 @@ import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; +import { dialChatWire, type ChatWire } from '../handoff.ts'; import { applyRoleCompatibilityToResponses, disableReasoningOnForcedToolChoiceForResponses, @@ -60,7 +65,7 @@ import { chatTargetPicker } from '../shared/target-picker.ts'; import { wrapResponsesAffinityEgress } from './affinity/egress.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; import { doneFrame, renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, @@ -190,21 +195,20 @@ const renderSSE = (frames: AsyncIterable>): }); /** - * The native wire. It dials Responses and provides the answer at this family's own response - * key — which is what will make it interchangeable with a translated chain: both hand up - * `response.chat.responses`, and the stage above cannot tell which ran. + * The wire. It dials Responses and provides the answer at whichever family's response key the + * chain above it reads — which is what makes it interchangeable with a translated chain: both + * hand up `response.chat.responses`, and the stage above cannot tell which ran. */ -const callResponsesUpstream = defineStage< +const callResponsesUpstream = (streamedUsage: string) => defineStage< R<'request.chat.responses' | 'route.attempt' | 'ingress.http.headers'>, - R<'response.chat.responses' | 'response.chat.responses.streamedUsage' - | 'response.usage.billable' | 'response.http.headers'>, + R<'response.chat.responses' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ name: 'callResponsesUpstream', return: { provides: [ 'response.chat.responses', - 'response.chat.responses.streamedUsage', + streamedUsage, 'response.usage.billable', 'response.http.headers', ], @@ -216,11 +220,12 @@ const callResponsesUpstream = defineStage< use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); // What the record holds by now: the payload affinity materialized for this candidate, as - // every stage between the fork and here has rewritten it. Client-carried state — an - // encrypted reasoning blob, a compaction the upstream issued — was rewritten for the - // upstream that will see it, which is the whole reason a turn can be pinned at all. The - // id the client addressed does not travel — the provider re-stamps whatever it resolved - // upstream — and an alias' own rules apply to the body that is sent. + // every stage between the fork and here has rewritten it — or, on a translated wire, what + // the handoff put here. Client-carried state — an encrypted reasoning blob, a compaction + // the upstream issued — was rewritten for the upstream that will see it, which is the + // whole reason a turn can be pinned at all. The id the client addressed does not travel — + // the provider re-stamps whatever it resolved upstream — and an alias' own rules apply to + // the body that is sent. // // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to, which is the one a wire takes. @@ -248,7 +253,7 @@ const callResponsesUpstream = defineStage< return move({ ...facts, 'response.chat.responses': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.chat.responses.streamedUsage': null, + [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], }); @@ -271,7 +276,7 @@ const callResponsesUpstream = defineStage< message: text, ...(parsed === undefined ? {} : { body: parsed }), }, - 'response.chat.responses.streamedUsage': null, + [streamedUsage]: null, 'response.usage.billable': called, 'response.http.headers': [...result.response.headers], }); @@ -289,7 +294,7 @@ const callResponsesUpstream = defineStage< return move({ ...facts, 'response.chat.responses': { kind: 'value' as const, body: result.result }, - 'response.chat.responses.streamedUsage': null, + [streamedUsage]: null, 'response.usage.billable': [billed(identity, billableUsageFromResponsesResult(result.result) ?? undefined)], 'response.http.headers': [], }); @@ -299,13 +304,37 @@ const callResponsesUpstream = defineStage< return move({ ...facts, 'response.chat.responses': { kind: 'stream' as const, frames: metered.frames }, - 'response.chat.responses.streamedUsage': metered.outcome, + [streamedUsage]: metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); }, }); +/** + * The Responses wire, as the chain that dials it. + * + * Every source protocol that reaches an upstream over this endpoint runs this, whether the + * client spoke Responses or a handoff arrived here — which is what makes the two rules above + * the dial belong here. The role rewrite states what an upstream's Responses endpoint + * accepts; the cache-bucket fold speaks about the usage *this* wire reports and about the + * flag that describes it, and a translator emits the canonical form, which is the one case + * the fold has nothing to do with. + */ +export const responsesWire = (streamedUsage: string): readonly Stage[] => [ + applyRoleCompatibilityToResponses, + normalizeExclusiveCachedTokensForResponses, + callResponsesUpstream(streamedUsage), +]; + +/** This family's own reading, which every wire under it hands up. */ +const STREAMED_USAGE = 'response.chat.responses.streamedUsage'; + +/** The wires `/v1/responses` can be served on. Only its own is built; the two translated ones + * are each a handoff and then that protocol's own wire, and until they exist a candidate the + * picker admitted on one of them is dialled here anyway. */ +const responsesWireFor = (): ChatWire => compose('responsesNative', responsesWire(STREAMED_USAGE)); + /** Reads the upstream's own usage off its own events as they pass, so the reading costs one * pass and the client's stream is what drives it. Responses states its counts on the * lifecycle envelopes, and only one carrying real counts replaces the running figure, so an @@ -418,10 +447,14 @@ export const responsesServePipeline = (payload: CanonicalResponsesPayload): Pipe }), materializeAttempt('request.chat.responses'), disableReasoningOnForcedToolChoiceForResponses, - applyRoleCompatibilityToResponses, stripPromptCacheKeyForResponses, - normalizeExclusiveCachedTokensForResponses, vendorDeepSeekNormalizeForResponses, vendorQwenNormalizeForResponses, - callResponsesUpstream, + dialChatWire({ + source: 'request.chat.responses', + needs: ['request.chat.responses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.responses', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => responsesTarget.pick(endpoints), + wire: responsesWireFor, + }), ]); From 9b8ae763c9a534e908a1fd1bc81fe5f70d40be10 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 07:56:17 +0800 Subject: [PATCH 035/156] feat(gemini): reach a candidate over any of the three wires a translation can take MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `:generateContent` has no wire of its own, so the picker's preference list is entirely translated — but only the first entry was built, and a candidate that `canServe` admitted on its Messages or Responses endpoint was dialled on Chat Completions anyway and refused there. Each of the two is what the first already is: a handoff into that protocol, followed by that protocol's own wire. The fork picks per candidate, so nothing else changes. `packages/gateway/__tests__/data-plane/chat/gemini/http_test.ts > POST /v1beta/models/:model:generateContent translates through Messages target end to end` covered this and had been failing; the matrix in `translated-wires_test.ts` states it per pair, on candidates that carry exactly one endpoint each so a chain that picked another wire would call a provider method that throws. --- .../data-plane/chat/translated-wires_test.ts | 225 ++++++++++++++++++ .../src/data-plane/chat/gemini/pipeline.ts | 62 +++-- 2 files changed, 266 insertions(+), 21 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts new file mode 100644 index 0000000000..8474a63d65 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -0,0 +1,225 @@ +// The translated wires, run end to end — one row per (source, target) pair. +// +// The property each row states is the same, and it is the one nothing else can state: a +// candidate reachable *only* over another protocol's endpoint is dialled there rather than +// refused or dialled on the wrong wire, and what comes back reaches the client in the +// client's own protocol. The candidates here carry exactly one endpoint each, so a chain that +// picked any other wire would call a provider method that throws. +// +// Gemini's three rows are here; the other families' fill in beside them. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run, type Pipeline } from '@floway-dev/pipeline'; +import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocols/common'; +import type { GeminiPayload, GeminiResult } from '@floway-dev/protocols/gemini'; +import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; +import type { ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +const MODEL = 'model-x'; + +let live: readonly ModelCandidate[] = []; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +interface Calls { + readonly callChatCompletions?: (model: unknown, body: unknown) => Promise>; + readonly callMessages?: (model: unknown, body: unknown) => Promise>; + readonly callResponses?: (model: unknown, body: unknown) => Promise; +} + +/** A candidate reachable over exactly the endpoints it is given. Handing it one endpoint is + * what pins the wire: every provider method the stub was not given throws, so a chain that + * picked another wire fails loudly rather than quietly answering. */ +const candidate = ( + upstreamId: string, + endpoints: ModelEndpoints, + calls: Calls, + flags: readonly FlagId[] = [], +): ModelCandidate => ({ + provider: { + upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider(calls as never), + }, + model: stubInternalModel( + { + id: MODEL, + endpoints, + providerModels: { [upstreamId]: stubProviderModel({ id: MODEL, endpoints, enabledFlags: new Set(flags) }) }, + }, + upstreamId, + ), + fetcher: directFetcher, +} as unknown as ModelCandidate); + +// ── What each wire answers with ─────────────────────────────────────────────────────────── + +const chatCompletionsTurn = (text: string): ProviderStreamResult => ({ + ok: true, + modelKey: 'k', + headers: new Headers(), + events: (async function* () { + yield eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: MODEL, + choices: [{ index: 0, delta: { role: 'assistant', content: text }, finish_reason: null }], + } as ChatCompletionsStreamEvent); + yield eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: MODEL, + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + } as ChatCompletionsStreamEvent); + yield doneFrame(); + })(), +}); + +const messagesTurn = (text: string): ProviderStreamResult => ({ + ok: true, + modelKey: 'k', + headers: new Headers(), + events: (async function* () { + yield eventFrame({ + type: 'message_start', + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: MODEL, + stop_reason: null, stop_sequence: null, usage: { input_tokens: 3, output_tokens: 0 }, + }, + } as MessagesStreamEvent); + yield eventFrame({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } } as MessagesStreamEvent); + yield eventFrame({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } } as MessagesStreamEvent); + yield eventFrame({ type: 'content_block_stop', index: 0 } as MessagesStreamEvent); + yield eventFrame({ type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 2 } } as MessagesStreamEvent); + yield eventFrame({ type: 'message_stop' } as MessagesStreamEvent); + yield doneFrame(); + })(), +}); + +const responsesResult = (text: string): ResponsesResult => ({ + id: 'resp_1', + object: 'response', + model: MODEL, + status: 'completed', + output: [{ + type: 'message', id: 'msg_1', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text, annotations: [] }], + }], + error: null, + incomplete_details: null, +}); + +const responsesTurn = (text: string): ProviderResponsesResult => ({ + action: 'generate', + ok: true, + modelKey: 'k', + headers: new Headers(), + events: (async function* () { + yield eventFrame({ + type: 'response.output_item.added', + sequence_number: 1, + output_index: 0, + item: { type: 'message', id: 'msg_1', role: 'assistant', status: 'in_progress', content: [] }, + } as ResponsesStreamEvent); + yield eventFrame({ + type: 'response.output_text.delta', + sequence_number: 2, item_id: 'msg_1', output_index: 0, content_index: 0, delta: text, + } as ResponsesStreamEvent); + yield eventFrame({ type: 'response.completed', sequence_number: 3, response: responsesResult(text) } as ResponsesStreamEvent); + yield doneFrame(); + })(), +}); + +// ── Running one source protocol's chain ─────────────────────────────────────────────────── + +const serve = async ( + pipeline: Pipeline, + facts: Record, + attemptPayload: unknown, +) => { + const gateway = mockChatGatewayCtx({ wantsStream: false }); + return await run(pipeline, move(facts) as never, { + gateway, + background: () => {}, + rememberCandidates: () => {}, + rememberChatSelection: () => {}, + chatPayloadFor: () => attemptPayload, + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never); +}; + +const geminiPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiPayload; + +const serveGemini = async (payload: GeminiPayload = geminiPayload) => + await serve(geminiServePipeline(payload), { + 'ingress.http.headers': [], + 'ingress.chat.sourceProtocol': 'gemini', + 'ingress.chat.gemini.wantsStream': false, + 'request.chat.gemini': payload, + 'serve.model': MODEL, + }, payload); + +// ── What the client was answered with, per protocol ─────────────────────────────────────── + +const geminiText = (rendered: unknown): string | undefined => + (rendered as GeminiResult).candidates?.[0]?.content.parts.map(part => part.text ?? '').join(''); + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('a chat family reaching a candidate over another protocol', () => { + it('serves :generateContent on a Chat Completions-only candidate', async () => { + const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); + resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); + + const { facts, drain } = await serveGemini(); + + expect(callChatCompletions).toHaveBeenCalledTimes(1); + expect(geminiText(facts['response.chat.gemini.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves :generateContent on a Messages-only candidate', async () => { + const callMessages = vi.fn(async () => messagesTurn('hello')); + resolves([candidate('up_a', { messages: {} }, { callMessages })]); + + const { facts, drain } = await serveGemini(); + + expect(callMessages).toHaveBeenCalledTimes(1); + expect(geminiText(facts['response.chat.gemini.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves :generateContent on a Responses-only candidate', async () => { + const callResponses = vi.fn(async () => responsesTurn('hello')); + resolves([candidate('up_a', { responses: {} }, { callResponses })]); + + const { facts, drain } = await serveGemini(); + + expect(callResponses).toHaveBeenCalledTimes(1); + expect(geminiText(facts['response.chat.gemini.rendered'])).toBe('hello'); + await drain(); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 05e261e1d2..2082168aec 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -13,9 +13,10 @@ // `response.chat.gemini`. // // It is also the one family with no wire of its own: the provider surface has no `callGemini`, -// so every candidate is reached through a translation and the wire is whichever of the three -// translatable ones the picker finds on it. Only the first, Chat Completions, is built here — -// a handoff into that protocol, followed by that protocol's own wire. +// so every candidate is reached through a translation and all three wires are translated ones +// — a handoff into Chat Completions, into Messages or into Responses, followed by that +// protocol's own wire. Which one a candidate is dialled on is the picker's answer for that +// candidate, and failover re-running the suffix is what re-asks for the next. // // Because every wire is translated, the reading is taken on the dialect the upstream actually // spoke: the target protocol's own wire meters its own frames on their way into the @@ -25,10 +26,6 @@ // reporting a finish reason translates into candidates that never finish. // // What is not built, stated rather than implied: -// - the Messages and the Responses wires. Each is a handoff of its own into that protocol's -// wire, and until they exist a candidate the picker admitted on one of those two is -// dialled on Chat Completions anyway, which is the same interim state the other three -// chains carry for their own translated wires. // - `:countTokens`. It is a second entry against this protocol rather than another wire of // this one: no stream, no billable turn, and a `{ totalTokens }` envelope of its own. // - the affinity egress' thought-signature rewrite is here, but the Gemini interceptors are @@ -60,6 +57,8 @@ import { stripUnsupportedToolsFromGemini, suppressThoughtPartsFromGemini, } from '../interceptors.ts'; +import { messagesWire } from '../messages/pipeline.ts'; +import { responsesWire } from '../responses/pipeline.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; @@ -75,7 +74,7 @@ import { type GeminiStreamEvent, } from '@floway-dev/protocols/gemini'; import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; -import { translateGeminiViaChatCompletions } from '@floway-dev/translate'; +import { translateGeminiViaChatCompletions, translateGeminiViaMessages, translateGeminiViaResponses } from '@floway-dev/translate'; /** `:generateContent` has no wire of its own, so the whole preference list is translated: * Chat Completions first, then Messages, then Responses. */ @@ -248,19 +247,40 @@ const framesUntilTerminal = async function* ( /** This family's own reading, which every wire under it hands up. */ const STREAMED_USAGE = 'response.chat.gemini.streamedUsage'; -/** The wires `:generateContent` can be served on, and every one of them is a translated one: - * the provider surface has no Gemini call, so a handoff is how every candidate is reached. - * Only the first preference is built; until the other two exist a candidate the picker - * admitted on one of them is dialled here anyway. */ -const geminiWireFor = (_target: ChatTargetApi, candidate: ModelCandidate): ChatWire => - compose('geminiViaChatCompletions', [ - handOff({ - from: { request: 'request.chat.gemini', response: 'response.chat.gemini' }, - to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, - trip: async payload => await translateGeminiViaChatCompletions(payload, { model: candidate.model.id }), - }), - ...chatCompletionsWire(STREAMED_USAGE), - ]); +/** The three wires `:generateContent` can be served on, and all three are translated ones: + * the provider surface has no Gemini call, so a handoff is how every candidate is reached. */ +const geminiWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { + const context = { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens }; + switch (target) { + case 'chat-completions': + return compose('geminiViaChatCompletions', [ + handOff({ + from: { request: 'request.chat.gemini', response: 'response.chat.gemini' }, + to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + trip: async payload => await translateGeminiViaChatCompletions(payload, context), + }), + ...chatCompletionsWire(STREAMED_USAGE), + ]); + case 'messages': + return compose('geminiViaMessages', [ + handOff({ + from: { request: 'request.chat.gemini', response: 'response.chat.gemini' }, + to: { request: 'request.chat.messages', response: 'response.chat.messages' }, + trip: async payload => await translateGeminiViaMessages(payload, context), + }), + ...messagesWire(STREAMED_USAGE), + ]); + case 'responses': + return compose('geminiViaResponses', [ + handOff({ + from: { request: 'request.chat.gemini', response: 'response.chat.gemini' }, + to: { request: 'request.chat.responses', response: 'response.chat.responses' }, + trip: async payload => await translateGeminiViaResponses(payload, context), + }), + ...responsesWire(STREAMED_USAGE), + ]); + } +}; /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is From 4906782b992892687510e85e6638a8d923cca38d Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 07:58:47 +0800 Subject: [PATCH 036/156] feat(chat-completions): reach a candidate over the translated Messages and Responses wires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/v1/chat/completions` prefers its own wire and then those two, but only the first was built, so a candidate whose model carries no `chatCompletions` endpoint was admitted by the picker and then dialled where it cannot answer. Each translated wire is a handoff into that protocol followed by that protocol's own wire, and hands up `response.chat.chatCompletions` like the native one does — so the stage above cannot tell which ran, and failover re-running the suffix is the whole of how the next candidate re-picks. That last part gets a test of its own: a refused native candidate and a Messages-only one behind it. --- .../data-plane/chat/translated-wires_test.ts | 69 ++++++++++++++++++- .../chat/chat-completions/pipeline.ts | 51 +++++++++++--- 2 files changed, 108 insertions(+), 12 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 8474a63d65..d2a983e7a1 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -6,16 +6,19 @@ // client's own protocol. The candidates here carry exactly one endpoint each, so a chain that // picked any other wire would call a provider method that throws. // -// Gemini's three rows are here; the other families' fill in beside them. +// Gemini's three rows are here, and Chat Completions' two; the remaining families' fill in +// beside them. Beside the matrix, one thing that is about the fork rather than about a pair: +// that failover moves from a native candidate onto a translated one. import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run, type Pipeline } from '@floway-dev/pipeline'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import type { ChatCompletionsPayload, ChatCompletionsResult, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocols/common'; import type { GeminiPayload, GeminiResult } from '@floway-dev/protocols/gemini'; import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; @@ -165,8 +168,19 @@ const serve = async ( } as never); }; +const chatCompletionsPayload = { model: MODEL, messages: [{ role: 'user', content: 'hi' }] } as unknown as ChatCompletionsPayload; const geminiPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiPayload; +const serveChatCompletions = async (payload: ChatCompletionsPayload = chatCompletionsPayload) => + await serve(chatCompletionsServePipeline(payload), { + 'ingress.http.headers': [], + 'ingress.chat.sourceProtocol': 'chatCompletions', + 'ingress.chat.chatCompletions.wantsStream': false, + 'ingress.chat.chatCompletions.wantsUsageChunk': false, + 'request.chat.chatCompletions': payload, + 'serve.model': MODEL, + }, payload); + const serveGemini = async (payload: GeminiPayload = geminiPayload) => await serve(geminiServePipeline(payload), { 'ingress.http.headers': [], @@ -178,6 +192,9 @@ const serveGemini = async (payload: GeminiPayload = geminiPayload) => // ── What the client was answered with, per protocol ─────────────────────────────────────── +const chatCompletionsText = (rendered: unknown): string | null | undefined => + (rendered as ChatCompletionsResult).choices[0]?.message.content as string | null | undefined; + const geminiText = (rendered: unknown): string | undefined => (rendered as GeminiResult).candidates?.[0]?.content.parts.map(part => part.text ?? '').join(''); @@ -190,6 +207,29 @@ beforeEach(() => { }); describe('a chat family reaching a candidate over another protocol', () => { + it('serves /v1/chat/completions on a Messages-only candidate', async () => { + const callMessages = vi.fn(async () => messagesTurn('hello')); + resolves([candidate('up_a', { messages: {} }, { callMessages })]); + + const { facts, drain } = await serveChatCompletions(); + + expect(callMessages).toHaveBeenCalledTimes(1); + expect(facts['response.http.status']).toBe(200); + expect(chatCompletionsText(facts['response.chat.chatCompletions.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves /v1/chat/completions on a Responses-only candidate', async () => { + const callResponses = vi.fn(async () => responsesTurn('hello')); + resolves([candidate('up_a', { responses: {} }, { callResponses })]); + + const { facts, drain } = await serveChatCompletions(); + + expect(callResponses).toHaveBeenCalledTimes(1); + expect(chatCompletionsText(facts['response.chat.chatCompletions.rendered'])).toBe('hello'); + await drain(); + }); + it('serves :generateContent on a Chat Completions-only candidate', async () => { const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); @@ -223,3 +263,28 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); }); + +describe('the fork over wires', () => { + // Failover is an ordinary middle stage: what it re-runs is the whole suffix including the + // stage that picks the wire, so the next candidate re-picks. Nothing here is a mechanism of + // failover's own — the first candidate goes native and the second goes translated because + // that is what each one's own endpoints say. + it('moves from a refused native wire onto a translated one', async () => { + const callChatCompletions = vi.fn(async () => ({ + ok: false as const, modelKey: 'k', response: Response.json({ error: { message: 'slow down' } }, { status: 429 }), + })); + const callMessages = vi.fn(async () => messagesTurn('second')); + resolves([ + candidate('up_a', { chatCompletions: {} }, { callChatCompletions }), + candidate('up_b', { messages: {} }, { callMessages }), + ]); + + const { facts, drain } = await serveChatCompletions(); + + expect(callChatCompletions).toHaveBeenCalledTimes(1); + expect(callMessages).toHaveBeenCalledTimes(1); + expect(facts['response.http.status']).toBe(200); + expect(chatCompletionsText(facts['response.chat.chatCompletions.rendered'])).toBe('second'); + await drain(); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index cbfa260ab0..6be6bc8fdc 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -8,9 +8,10 @@ // dialChatWire the ending: picks this candidate's wire and hands into it // // The wire a candidate is dialled on is chosen per candidate rather than at assembly, and -// because failover re-runs the whole suffix the next candidate re-picks its own. Only its own -// wire is built here; the translated ones will hand up `response.chat.chatCompletions` too, -// which is what will make them interchangeable with it. +// because failover re-runs the whole suffix the next candidate re-picks its own. Its own +// wire is a bare ending; the two translated ones are a handoff and then that protocol's own +// wire, and all three hand up `response.chat.chatCompletions` — so the stage above cannot +// tell which ran. // // What sits *in* a wire rather than above the fork is the other half of the arrangement. A // rule that speaks about an upstream's Chat Completions endpoint — the role rewrite does, @@ -32,18 +33,21 @@ import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; -import { dialChatWire, type ChatWire } from '../handoff.ts'; +import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { applyRoleCompatibilityToChatCompletions, disableReasoningOnForcedToolChoiceForChatCompletions, stripPromptCacheKeyForChatCompletions, } from '../interceptors.ts'; +import { messagesWire } from '../messages/pipeline.ts'; +import { responsesWire } from '../responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; +import { createExternalImageLoader } from '../shared/external-image-loader.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; +import { compose, defineStage, move, type Pipeline, type Stage, type Use } from '@floway-dev/pipeline'; import { CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, chatCompletionsErrorPayloadMessage, @@ -53,7 +57,8 @@ import { type ChatCompletionsStreamEvent, } from '@floway-dev/protocols/chat-completions'; import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; -import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { translateChatCompletionsViaMessages, translateChatCompletionsViaResponses } from '@floway-dev/translate'; /** `/v1/chat/completions` prefers its own wire, then the translated Messages path, then the * translated Responses path. */ @@ -288,10 +293,36 @@ export const chatCompletionsWire = (streamedUsage: string): readonly Stage[] => /** This family's own reading, which every wire under it hands up. */ const STREAMED_USAGE = 'response.chat.chatCompletions.streamedUsage'; -/** The wires `/v1/chat/completions` can be served on. Only its own is built; the two - * translated ones are each a handoff and then that protocol's own wire, and until they exist - * a candidate the picker admitted on one of them is dialled here anyway. */ -const chatCompletionsWireFor = (): ChatWire => compose('chatCompletionsNative', chatCompletionsWire(STREAMED_USAGE)); +/** The three wires `/v1/chat/completions` can be served on. Its own is the bare wire; each + * translated one is a handoff and then the target protocol's own wire. */ +const chatCompletionsWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { + switch (target) { + case 'chat-completions': + return compose('chatCompletionsNative', chatCompletionsWire(STREAMED_USAGE)); + case 'messages': + return compose('chatCompletionsViaMessages', [ + handOff({ + from: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + to: { request: 'request.chat.messages', response: 'response.chat.messages' }, + trip: async payload => await translateChatCompletionsViaMessages(payload, { + model: candidate.model.id, + fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, + loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), + }), + }), + ...messagesWire(STREAMED_USAGE), + ]); + case 'responses': + return compose('chatCompletionsViaResponses', [ + handOff({ + from: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + to: { request: 'request.chat.responses', response: 'response.chat.responses' }, + trip: async payload => await translateChatCompletionsViaResponses(payload, { model: candidate.model.id }), + }), + ...responsesWire(STREAMED_USAGE), + ]); + } +}; /** Reads the upstream's own usage off its own events as they pass, so the reading costs one * pass and the client's stream is what drives it. Only a report carrying real counts From da3645c934c05ea83a734f754f2a8bcd0fac6d1d Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 08:01:45 +0800 Subject: [PATCH 037/156] feat(messages): reach a candidate over the translated Responses and Chat Completions wires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/v1/messages` prefers its own wire and then those two, but only the first was built, so a candidate the picker admitted on its Responses or Chat Completions endpoint was dialled where it cannot answer. Each translated wire is a handoff into that protocol followed by that protocol's own wire. The `messages-via-*` pairs are the two that declare an `apiError`, so this is also where the refusal rewrite arrives: an OpenAI-shaped context-window error becomes the Anthropic `prompt is too long:` shape Claude Code reads for auto-compaction, and a refusal the pair says nothing about reaches the client as the upstream wrote it. The system-role rewrite gets its own row too. It states what an upstream's Messages endpoint accepts, so a turn that leaves for Chat Completions must be folded by that protocol's rule instead — which the two folds disagree about for a leading system message, and that is what the test reads. `compose` says the other half of it: the rule needs `request.chat.messages`, so it cannot be placed in a chain where a handoff has consumed it. --- .../data-plane/chat/translated-wires_test.ts | 148 +++++++++++++++++- .../src/data-plane/chat/messages/pipeline.ts | 51 ++++-- 2 files changed, 182 insertions(+), 17 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index d2a983e7a1..741dbb0c69 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -6,22 +6,27 @@ // client's own protocol. The candidates here carry exactly one endpoint each, so a chain that // picked any other wire would call a provider method that throws. // -// Gemini's three rows are here, and Chat Completions' two; the remaining families' fill in -// beside them. Beside the matrix, one thing that is about the fork rather than about a pair: -// that failover moves from a native candidate onto a translated one. +// Gemini's three rows are here, Chat Completions' two, and Messages' two; the remaining +// family's fill in beside them. Beside the matrix, three things that are about the fork rather +// than about a pair: that failover moves from a native candidate onto a translated one, that a +// pair's own refusal rewrite survives the trip, and that a rule which speaks about one +// protocol's wire does not run on a turn that leaves for another — which is what the +// interceptor form said with `ctx.targetApi !== ` and what position says here. import { beforeEach, describe, expect, it, vi } from 'vitest'; import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; +import { handOff } from '../../../src/data-plane/chat/handoff.ts'; +import { messagesServePipeline, messagesWire } from '../../../src/data-plane/chat/messages/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; -import { move, run, type Pipeline } from '@floway-dev/pipeline'; +import { compose, move, run, type Pipeline } from '@floway-dev/pipeline'; import type { ChatCompletionsPayload, ChatCompletionsResult, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocols/common'; import type { GeminiPayload, GeminiResult } from '@floway-dev/protocols/gemini'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; +import { PROMPT_TOO_LONG_MESSAGE, type MessagesPayload, type MessagesResult, type MessagesStreamEvent } from '@floway-dev/protocols/messages'; import type { ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -145,6 +150,16 @@ const responsesTurn = (text: string): ProviderResponsesResult => ({ })(), }); +/** A refusal, in the words an OpenAI-shaped upstream uses for a turn that will not fit. */ +const contextExceeded = async (): Promise> => ({ + ok: false, + modelKey: 'k', + response: Response.json( + { error: { code: 'context_length_exceeded', type: 'invalid_request_error', message: "This model's maximum context length is 8192 tokens" } }, + { status: 400 }, + ), +}); + // ── Running one source protocol's chain ─────────────────────────────────────────────────── const serve = async ( @@ -169,6 +184,7 @@ const serve = async ( }; const chatCompletionsPayload = { model: MODEL, messages: [{ role: 'user', content: 'hi' }] } as unknown as ChatCompletionsPayload; +const messagesPayload = { model: MODEL, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as MessagesPayload; const geminiPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiPayload; const serveChatCompletions = async (payload: ChatCompletionsPayload = chatCompletionsPayload) => @@ -181,6 +197,15 @@ const serveChatCompletions = async (payload: ChatCompletionsPayload = chatComple 'serve.model': MODEL, }, payload); +const serveMessages = async (payload: MessagesPayload = messagesPayload) => + await serve(messagesServePipeline(payload), { + 'ingress.http.headers': [], + 'ingress.chat.sourceProtocol': 'messages', + 'ingress.chat.messages.wantsStream': false, + 'request.chat.messages': payload, + 'serve.model': MODEL, + }, payload); + const serveGemini = async (payload: GeminiPayload = geminiPayload) => await serve(geminiServePipeline(payload), { 'ingress.http.headers': [], @@ -195,6 +220,13 @@ const serveGemini = async (payload: GeminiPayload = geminiPayload) => const chatCompletionsText = (rendered: unknown): string | null | undefined => (rendered as ChatCompletionsResult).choices[0]?.message.content as string | null | undefined; +/** The answer's own text, past the carrier the edge writes back: on this protocol the turn's + * state rides as a redacted-thinking block at the head of the content. */ +const messagesText = (rendered: unknown): string | undefined => { + const block = (rendered as MessagesResult).content.find(part => part.type === 'text'); + return block?.type === 'text' ? block.text : undefined; +}; + const geminiText = (rendered: unknown): string | undefined => (rendered as GeminiResult).candidates?.[0]?.content.parts.map(part => part.text ?? '').join(''); @@ -230,6 +262,28 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); + it('serves /v1/messages on a Responses-only candidate', async () => { + const callResponses = vi.fn(async () => responsesTurn('hello')); + resolves([candidate('up_a', { responses: {} }, { callResponses })]); + + const { facts, drain } = await serveMessages(); + + expect(callResponses).toHaveBeenCalledTimes(1); + expect(messagesText(facts['response.chat.messages.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves /v1/messages on a Chat Completions-only candidate', async () => { + const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); + resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); + + const { facts, drain } = await serveMessages(); + + expect(callChatCompletions).toHaveBeenCalledTimes(1); + expect(messagesText(facts['response.chat.messages.rendered'])).toBe('hello'); + await drain(); + }); + it('serves :generateContent on a Chat Completions-only candidate', async () => { const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); @@ -287,4 +341,88 @@ describe('the fork over wires', () => { expect(chatCompletionsText(facts['response.chat.chatCompletions.rendered'])).toBe('second'); await drain(); }); + + // The canonical rewrite: Claude Code reads Anthropic's `prompt is too long:` shape to know + // it must auto-compact, and an OpenAI-shaped upstream states the same condition in its own + // words. The pair owns the translation of that refusal, and the handoff is what asks it. + it('rewrites a context-window refusal into the pair-s own envelope', async () => { + resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions: contextExceeded })]); + + const { facts, drain } = await serveMessages(); + + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.messages.rendered']).toEqual({ + type: 'error', + error: { type: 'invalid_request_error', message: PROMPT_TOO_LONG_MESSAGE }, + }); + await drain(); + }); + + // A refusal a pair says nothing about reaches the client as the upstream wrote it, with the + // status the upstream sent. + it('hands a refusal the pair does not speak about up unchanged', async () => { + const callChatCompletions = vi.fn(async () => ({ + ok: false as const, modelKey: 'k', response: Response.json({ error: { message: 'no' } }, { status: 403 }), + })); + resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); + + const { facts, drain } = await serveMessages(); + + expect(facts['response.http.status']).toBe(403); + expect(facts['response.chat.messages.rendered']).toEqual({ error: { message: 'no' } }); + await drain(); + }); +}); + +describe('a rule that speaks about one protocol-s wire', () => { + // The Messages fold rewrites *every* inline system message, because Anthropic's top-level + // `system` is the only first-position slot there. The Chat Completions fold rewrites only a + // system message past the leading run. So a leading system message tells the two apart: a + // turn that left for Chat Completions must be folded by the Chat Completions rule, which + // leaves it alone, and not by the Messages one, which would not. + it('does not run on a turn that leaves for another protocol', async () => { + let sent: { messages: { role: string }[] } | undefined; + const payload = { + model: MODEL, + max_tokens: 64, + messages: [{ role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }], + } as unknown as MessagesPayload; + resolves([candidate( + 'up_a', + { chatCompletions: {} }, + { + callChatCompletions: async (_model, body) => { + sent = body as { messages: { role: string }[] }; + return chatCompletionsTurn('hello'); + }, + }, + ['rewrite-mid-conv-system-to-user'], + )]); + + const { drain } = await serveMessages(payload); + + expect(sent?.messages.map(message => message.role)).toEqual(['system', 'user']); + await drain(); + }); + + // The other half, and this one assembly says rather than a run: the wire's own rules read + // the wire's own request key, so `compose` refuses to place them where a handoff has taken + // it. A chain cannot be arranged to run the Messages rule on a turn on its way out of + // Messages, whatever anyone writes. + it('cannot be placed in a chain whose payload a handoff consumed', () => { + const leaving = handOff({ + from: { request: 'request.chat.messages', response: 'response.chat.messages' }, + to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + trip: () => { throw new Error('not dialled'); }, + }); + const arriving = handOff({ + from: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + to: { request: 'request.chat.messages', response: 'response.chat.messages' }, + trip: () => { throw new Error('not dialled'); }, + }); + + expect(() => compose('leaving', [leaving, ...messagesWire('r')])) + .toThrow(/needs request\.chat\.messages, which handOff:.* consumed above it/); + expect(() => compose('arriving', [arriving, ...messagesWire('r')])).not.toThrow(); + }); }); diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index 6522db65b4..288b657354 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -7,12 +7,14 @@ // materializeAttempt puts the payload this candidate is owed into the record // dialChatWire the ending: picks this candidate's wire and hands into it // -// Only its own wire is built. The translated ones — Messages via Responses and Messages via -// Chat Completions — will hand up `response.chat.messages` too, which is what will make them -// interchangeable with it, and each is a step of its own. So is -// `/v1/messages/count_tokens`: a second operation over this protocol rather than another wire -// under this pipeline. This family's own interceptors are not all stages yet either, so the -// array between the materialized payload and the fork is short rather than complete. +// Three wires, all handing up `response.chat.messages`: this protocol's own, and the two +// translated ones — Messages via Responses and Messages via Chat Completions — each a +// handoff followed by that protocol's own wire. The stage above cannot tell which ran. +// +// `/v1/messages/count_tokens` is not one of them: it is a second operation over this +// protocol rather than another wire under this pipeline, and it is its own step. This +// family's own interceptors are not all stages yet either, so the array between the +// materialized payload and the fork is short rather than complete. import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { renderMessagesError } from './errors.ts'; @@ -26,13 +28,15 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import { chatCompletionsWire } from '../chat-completions/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; -import { dialChatWire, type ChatWire } from '../handoff.ts'; +import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { applyRoleCompatibilityToMessages, disableReasoningOnForcedToolChoiceForMessages, stripBillingAttributionFromMessages, } from '../interceptors.ts'; +import { responsesWire } from '../responses/pipeline.ts'; import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; @@ -50,7 +54,8 @@ import { type MessagesPayload, type MessagesStreamEvent, } from '@floway-dev/protocols/messages'; -import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { translateMessagesViaChatCompletions, translateMessagesViaResponses } from '@floway-dev/translate'; /** `/v1/messages` prefers its own wire, then the translated Responses path, then the * translated Chat Completions path. */ @@ -356,10 +361,32 @@ export const messagesWire = (streamedUsage: string): readonly Stage[] => [ /** This family's own reading, which every wire under it hands up. */ const STREAMED_USAGE = 'response.chat.messages.streamedUsage'; -/** The wires `/v1/messages` can be served on. Only its own is built; the two translated ones - * are each a handoff and then that protocol's own wire, and until they exist a candidate the - * picker admitted on one of them is dialled here anyway. */ -const messagesWireFor = (): ChatWire => compose('messagesNative', messagesWire(STREAMED_USAGE)); +/** The three wires `/v1/messages` can be served on. Its own is the bare wire; each translated + * one is a handoff and then the target protocol's own wire. */ +const messagesWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { + switch (target) { + case 'messages': + return compose('messagesNative', messagesWire(STREAMED_USAGE)); + case 'responses': + return compose('messagesViaResponses', [ + handOff({ + from: { request: 'request.chat.messages', response: 'response.chat.messages' }, + to: { request: 'request.chat.responses', response: 'response.chat.responses' }, + trip: async payload => await translateMessagesViaResponses(payload, { model: candidate.model.id }), + }), + ...responsesWire(STREAMED_USAGE), + ]); + case 'chat-completions': + return compose('messagesViaChatCompletions', [ + handOff({ + from: { request: 'request.chat.messages', response: 'response.chat.messages' }, + to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + trip: async payload => await translateMessagesViaChatCompletions(payload, { model: candidate.model.id }), + }), + ...chatCompletionsWire(STREAMED_USAGE), + ]); + } +}; /** Reads the upstream's own usage off its own events as they pass, so the reading costs one * pass and the client's stream is what drives it. Anthropic states input accounting on From a62126f1af8c4cfa2104cdcd9ec11179ee5a4973 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 08:03:19 +0800 Subject: [PATCH 038/156] feat(responses): reach a candidate over the translated Messages and Chat Completions wires `/v1/responses` prefers its own wire and then those two, but only the first was built, so a candidate the picker admitted on its Messages or Chat Completions endpoint was dialled where it cannot answer. Each translated wire is a handoff into that protocol followed by that protocol's own wire, which completes the matrix: every (source, target) pair the four chat families can be served on now has a row in `translated-wires_test.ts`. `/v1/responses/compact` is not among them. It stays on `responsesServe.compact`, where the shim that pivots compact into generate on a non-Responses target still lives; the dial here asks for `generate` and a handoff refuses to carry the single envelope a compaction answers with, because no pair maps one. --- .../data-plane/chat/translated-wires_test.ts | 53 ++++++++++++++--- .../src/data-plane/chat/responses/pipeline.ts | 57 ++++++++++++++----- 2 files changed, 89 insertions(+), 21 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 741dbb0c69..dab73cb7b7 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -6,12 +6,11 @@ // client's own protocol. The candidates here carry exactly one endpoint each, so a chain that // picked any other wire would call a provider method that throws. // -// Gemini's three rows are here, Chat Completions' two, and Messages' two; the remaining -// family's fill in beside them. Beside the matrix, three things that are about the fork rather -// than about a pair: that failover moves from a native candidate onto a translated one, that a -// pair's own refusal rewrite survives the trip, and that a rule which speaks about one -// protocol's wire does not run on a turn that leaves for another — which is what the -// interceptor form said with `ctx.targetApi !== ` and what position says here. +// Beside the matrix, three things that are about the fork rather than about a pair: that +// failover moves from a native candidate onto a translated one, that a pair's own refusal +// rewrite survives the trip, and that a rule which speaks about one protocol's wire does not +// run on a turn that leaves for another — which is what the interceptor form said with +// `ctx.targetApi !== ` and what position says here. import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -19,6 +18,7 @@ import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat- import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; import { handOff } from '../../../src/data-plane/chat/handoff.ts'; import { messagesServePipeline, messagesWire } from '../../../src/data-plane/chat/messages/pipeline.ts'; +import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; @@ -27,7 +27,7 @@ import type { ChatCompletionsPayload, ChatCompletionsResult, ChatCompletionsStre import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocols/common'; import type { GeminiPayload, GeminiResult } from '@floway-dev/protocols/gemini'; import { PROMPT_TOO_LONG_MESSAGE, type MessagesPayload, type MessagesResult, type MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { CanonicalResponsesPayload, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -185,6 +185,7 @@ const serve = async ( const chatCompletionsPayload = { model: MODEL, messages: [{ role: 'user', content: 'hi' }] } as unknown as ChatCompletionsPayload; const messagesPayload = { model: MODEL, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as MessagesPayload; +const responsesPayload = { model: MODEL, input: [{ type: 'message', role: 'user', content: 'hi' }] } as unknown as CanonicalResponsesPayload; const geminiPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiPayload; const serveChatCompletions = async (payload: ChatCompletionsPayload = chatCompletionsPayload) => @@ -206,6 +207,15 @@ const serveMessages = async (payload: MessagesPayload = messagesPayload) => 'serve.model': MODEL, }, payload); +const serveResponses = async (payload: CanonicalResponsesPayload = responsesPayload) => + await serve(responsesServePipeline(payload), { + 'ingress.http.headers': [], + 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.responses.wantsStream': false, + 'request.chat.responses': payload, + 'serve.model': MODEL, + }, payload); + const serveGemini = async (payload: GeminiPayload = geminiPayload) => await serve(geminiServePipeline(payload), { 'ingress.http.headers': [], @@ -227,6 +237,12 @@ const messagesText = (rendered: unknown): string | undefined => { return block?.type === 'text' ? block.text : undefined; }; +const responsesText = (rendered: unknown): string | undefined => { + const item = (rendered as ResponsesResult).output.find(output => output.type === 'message'); + const part = item?.type === 'message' ? item.content[0] : undefined; + return part?.type === 'output_text' ? part.text : undefined; +}; + const geminiText = (rendered: unknown): string | undefined => (rendered as GeminiResult).candidates?.[0]?.content.parts.map(part => part.text ?? '').join(''); @@ -284,6 +300,29 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); + it('serves /v1/responses on a Messages-only candidate', async () => { + const callMessages = vi.fn(async () => messagesTurn('hello')); + resolves([candidate('up_a', { messages: {} }, { callMessages })]); + + const { facts, drain } = await serveResponses(); + + expect(callMessages).toHaveBeenCalledTimes(1); + expect(responsesText(facts['response.chat.responses.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves /v1/responses on a Chat Completions-only candidate', async () => { + const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); + resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); + + const { facts, drain } = await serveResponses(); + + expect(callChatCompletions).toHaveBeenCalledTimes(1); + expect(responsesText(facts['response.chat.responses.rendered'])).toBe('hello'); + await drain(); + }); + + // Gemini has no wire of its own, so all three of its rows are translated ones. it('serves :generateContent on a Chat Completions-only candidate', async () => { const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index cc36879c21..7ad57f299f 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -7,10 +7,12 @@ // materializeAttempt puts the payload this candidate is owed into the record // dialChatWire the ending: picks this candidate's wire and hands into it // -// Only its own wire is built. What sits *in* a wire rather than above the fork is a rule that -// speaks about that wire: the role rewrite and the cache-bucket fold both do, which is why -// their interceptor forms stood down on `ctx.targetApi !== 'responses'`, and position says it -// here instead. +// Three wires, all handing up `response.chat.responses`: this protocol's own, and the two +// translated ones — Responses via Messages and Responses via Chat Completions — each a +// handoff followed by that protocol's own wire. What sits *in* a wire rather than above the +// fork is a rule that speaks about that wire: the role rewrite and the cache-bucket fold both +// do, which is why their interceptor forms stood down on `ctx.targetApi !== 'responses'`, and +// position says it here instead. // // One transport, and everything else this protocol owns is a step of its own, stated here // rather than implied by its absence: @@ -21,9 +23,6 @@ // wire under this pipeline, so it stays on `responsesServe.compact`. The dial here asks // for `generate`; what the ending answers with is the branch the provider says it ran, // which is why the envelope a compaction is arrives somewhere rather than nowhere. -// - the translated wires, Responses via Messages and via Chat Completions. Each will hand -// up `response.chat.responses` too, which is what will make them interchangeable with -// this one. // - this family's remaining interceptors, the server-tool shims among them. Still only in // the interceptor form, so the array between the materialized payload and the fork is // short rather than complete. @@ -49,8 +48,9 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import { chatCompletionsWire } from '../chat-completions/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; -import { dialChatWire, type ChatWire } from '../handoff.ts'; +import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { applyRoleCompatibilityToResponses, disableReasoningOnForcedToolChoiceForResponses, @@ -59,13 +59,15 @@ import { vendorDeepSeekNormalizeForResponses, vendorQwenNormalizeForResponses, } from '../interceptors.ts'; +import { messagesWire } from '../messages/pipeline.ts'; import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; +import { createExternalImageLoader } from '../shared/external-image-loader.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { wrapResponsesAffinityEgress } from './affinity/egress.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; +import { compose, defineStage, move, type Pipeline, type Stage, type Use } from '@floway-dev/pipeline'; import { doneFrame, renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, @@ -75,7 +77,8 @@ import { type CanonicalResponsesPayload, type ResponsesStreamEvent, } from '@floway-dev/protocols/responses'; -import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { translateResponsesViaChatCompletions, translateResponsesViaMessages } from '@floway-dev/translate'; /** `/v1/responses` prefers its own wire, then the translated Messages path, then the * translated Chat Completions path. */ @@ -330,10 +333,36 @@ export const responsesWire = (streamedUsage: string): readonly Stage[] => [ /** This family's own reading, which every wire under it hands up. */ const STREAMED_USAGE = 'response.chat.responses.streamedUsage'; -/** The wires `/v1/responses` can be served on. Only its own is built; the two translated ones - * are each a handoff and then that protocol's own wire, and until they exist a candidate the - * picker admitted on one of them is dialled here anyway. */ -const responsesWireFor = (): ChatWire => compose('responsesNative', responsesWire(STREAMED_USAGE)); +/** The three wires `/v1/responses` can be served on. Its own is the bare wire; each translated + * one is a handoff and then the target protocol's own wire. */ +const responsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { + switch (target) { + case 'responses': + return compose('responsesNative', responsesWire(STREAMED_USAGE)); + case 'messages': + return compose('responsesViaMessages', [ + handOff({ + from: { request: 'request.chat.responses', response: 'response.chat.responses' }, + to: { request: 'request.chat.messages', response: 'response.chat.messages' }, + trip: async payload => await translateResponsesViaMessages(payload, { + model: candidate.model.id, + fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, + loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), + }), + }), + ...messagesWire(STREAMED_USAGE), + ]); + case 'chat-completions': + return compose('responsesViaChatCompletions', [ + handOff({ + from: { request: 'request.chat.responses', response: 'response.chat.responses' }, + to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + trip: async payload => await translateResponsesViaChatCompletions(payload, { model: candidate.model.id }), + }), + ...chatCompletionsWire(STREAMED_USAGE), + ]); + } +}; /** Reads the upstream's own usage off its own events as they pass, so the reading costs one * pass and the client's stream is what drives it. Responses states its counts on the From d53534ddf463cb1b3463ed6e69a51e9cf0c46825 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 08:47:41 +0800 Subject: [PATCH 039/156] feat(responses): run the stored-items membrane as stages of the chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The membrane wrapped the chain rather than sitting in it. `serve-prep.ts` expanded `previous_response_id`, read the stored rows back into the payload and staged what this turn adds; `client-output.ts` stamped the response id this gateway owns, stored each item that closed, and completed the response resource. Both halves ran outside `responsesServePipeline`, so a chain that answered a request answered it with the upstream's own envelope id, nothing persisted, and no snapshot for the next turn to continue from. The request half is `hydrateStoredItems`, above the resolver because everything after it routes on what it produces: affinity has to read the hydrated payload, since a continuation carries the prior turn's state on items the client never sent. The two refusals it holds are its own — the verbatim `previous_response_not_found` 400 and the `item-not-found` 404 — in the shape every other refusal in this chain takes. The prepared payload reaches the resolver through the run's own cell, because a narrowing is built at assembly, before any fact exists. The response half is the edge's, where the frames still exist: `emitResponses` runs the whole egress — the affinity carrier it already ran, plus the three layers around it — before it folds, so a client that did not ask to stream is answered with the object the persisted frames add up to. With the membrane below it a turn can now fail after the upstream succeeded, so the edge gained the two endings `respond.ts` had: a 502 carrying the fault while nothing has gone out, and, once the status has, the protocol's own `error` and `response.failed` pair in place of the `[DONE]` that would say the turn finished. `request.chat.responses.privatePayloads` carries what the hydrated rows held privately and `beginStoredAttempt` reseeds the store's scratchpad from it once per attempt. Nothing writes to that scratchpad in a pipelined turn yet — the server-tool shim is what writes, and it is still an interceptor. --- .../chat/responses-pipeline_test.ts | 52 ++- .../chat/responses/client-output.ts | 11 +- .../src/data-plane/chat/responses/pipeline.ts | 342 +++++++++++++++--- .../src/data-plane/chat/responses/respond.ts | 7 +- 4 files changed, 344 insertions(+), 68 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index a49a265a81..507bf10e55 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -2,9 +2,11 @@ // down here is what only running it can say — that the edge terminates the client's stream // on `[DONE]` however the upstream's ended, that it folds a stream into one response object // when the client did not ask to stream, that it hands the turn's own state back for the -// client to carry, that a turn the upstream answered with one envelope instead of a stream is -// served as that envelope, that a refusal keeps the upstream's own status and words, and that -// a dial nobody answered is a value the fork can move past. +// client to carry, that the stored-items membrane's other half runs there too so the client +// is answered under an envelope this gateway minted, that a turn the upstream answered with +// one envelope instead of a stream is served as that envelope, that a refusal keeps the +// upstream's own status and words, and that a dial nobody answered is a value the fork can +// move past. import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -218,14 +220,24 @@ describe('the responses chain', () => { 'response.completed', undefined, ]); - expect(frames.map(frame => frame.data)).toEqual([ + expect(frames.slice(0, 2).map(frame => frame.data)).toEqual([ JSON.stringify(delta('he')), JSON.stringify(delta('llo')), - // The carrier's own two frames went out ahead of this one under two sequence numbers of - // their own, so the terminal event is numbered where they left it. - JSON.stringify({ ...completed('hello'), sequence_number: 4 }), - '[DONE]', ]); + expect(frames[3]!.data).toBe('[DONE]'); + + // The terminal event is the upstream's, restated: the membrane at the edge stamps the + // envelope id this gateway minted for the turn and completes the resource to what the + // schema requires of it, so the deltas ride through untouched and this one does not. + const terminal = JSON.parse(frames[2]!.data) as { sequence_number: number; response: ResponsesResult }; + // The carrier's own two frames went out ahead of it under two sequence numbers of their + // own, so the terminal event is numbered where they left it. + expect(terminal.sequence_number).toBe(4); + expect(terminal.response).toMatchObject({ + status: 'completed', + output: [{ content: [{ type: 'output_text', text: 'hello' }] }], + }); + expect(terminal.response.id).not.toBe('resp_1'); }); // The other half of affinity: the resolver reads client-carried state on the way down, and @@ -304,13 +316,24 @@ describe('the responses chain', () => { // A stream that ran out before its terminal event is a turn nobody can answer from: the // response was never stated complete, incomplete or failed, so serving what did arrive - // would report a truncated answer as a whole one. - it('fails a stream that ended without its terminal event', async () => { + // would report a truncated answer as a whole one. By then the client is already being + // streamed to and the status went out with the headers, so it is told in the protocol's own + // words — and the stream ends on that rather than on the terminator, because a stream that + // ended on `[DONE]` is a stream that finished. + it('tells a client already being streamed to that the stream never ended', async () => { resolves([candidate(async () => stream(delta('hi')))]); const { facts } = await serve(true); + const frames = await drain(facts['response.chat.responses.rendered']); - await expect(drain(facts['response.chat.responses.rendered'])).rejects.toThrow(RESPONSES_MISSING_TERMINAL_MESSAGE); + expect(frames.map(frame => frame.event)).toEqual(['response.output_text.delta', 'error']); + expect(JSON.parse(frames[1]!.data)).toMatchObject({ + type: 'error', + error: { message: RESPONSES_MISSING_TERMINAL_MESSAGE }, + }); + // The client was answered, and the run still failed: what the frames said on the way out + // is not what the row says the turn was. + expect((await facts['response.chat.responses.streamedUsage']!).failed).toBe(true); }); // The upstream speaks SSE whatever the client asked for, so a client that did not ask to @@ -321,11 +344,14 @@ describe('the responses chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(200); - expect(withoutCarrierItem(facts['response.chat.responses.rendered'] as unknown as ResponsesResult)).toMatchObject({ - id: 'resp_1', + const answered = withoutCarrierItem(facts['response.chat.responses.rendered'] as unknown as ResponsesResult); + expect(answered).toMatchObject({ status: 'completed', output: [{ content: [{ type: 'output_text', text: 'hello' }] }], }); + // One client response can span several upstream calls, so the envelope it is answered + // under is the one the membrane minted rather than the one the upstream sent. + expect(answered.id).not.toBe('resp_1'); }); // A compaction is one envelope rather than a stream: the upstream ran the turn, charged for diff --git a/packages/gateway/src/data-plane/chat/responses/client-output.ts b/packages/gateway/src/data-plane/chat/responses/client-output.ts index 2b85a8053b..8a1b04a7eb 100644 --- a/packages/gateway/src/data-plane/chat/responses/client-output.ts +++ b/packages/gateway/src/data-plane/chat/responses/client-output.ts @@ -34,14 +34,11 @@ export const wrapResponsesStatefulOutput = ( // at the stateful half and completes that resource itself. export const wrapResponsesClientEgress = ( frames: AsyncIterable>, - ctx: GatewayCtx, + ctx: ChatGatewayCtx, request: CanonicalResponsesPayload, -): AsyncIterable> => { - if (!('affinity' in ctx) || !('store' in ctx)) throw new Error('Responses output requires chat context'); - const chatCtx = ctx as ChatGatewayCtx; - return wrapResponseResourceCompletion(wrapResponsesStatefulOutput(frames, chatCtx), { +): AsyncIterable> => + wrapResponseResourceCompletion(wrapResponsesStatefulOutput(frames, ctx), { request, createdAt: responsesCreatedAt(ctx), - stored: chatCtx.store.writesState, + stored: ctx.store.writesState, }); -}; diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index 7ad57f299f..5dc50fd226 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -2,9 +2,11 @@ // // emitResponses the edge: writes the answer in the shape the client asked for // writeSettlement above the fork, so a run bills once however many wires it tried +// hydrateStoredItems the stored-items membrane, on the way in // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record +// beginStoredAttempt reseeds the store's per-attempt scratchpad // dialChatWire the ending: picks this candidate's wire and hands into it // // Three wires, all handing up `response.chat.responses`: this protocol's own, and the two @@ -18,26 +20,26 @@ // rather than implied by its absence: // // - the WebSocket transport. A second entry against this protocol — its own framing, its -// own lifecycle — and `websocket.ts` still owns it. +// own lifecycle — and `websocket.ts` still owns it. It reaches the same membrane through +// `serve-prep.ts` and `client-output.ts`, which is why both outlive this chain. // - `/v1/responses/compact`. A second operation over this protocol rather than another // wire under this pipeline, so it stays on `responsesServe.compact`. The dial here asks // for `generate`; what the ending answers with is the branch the provider says it ran, // which is why the envelope a compaction is arrives somewhere rather than nowhere. // - this family's remaining interceptors, the server-tool shims among them. Still only in // the interceptor form, so the array between the materialized payload and the fork is -// short rather than complete. -// - the stored-items membrane: `previous_response_id` expansion, item hydration, and the -// snapshot the next turn reads. It wraps the chain rather than sitting in it, and -// `serve-prep.ts` and `client-output.ts` still hold it. +// short rather than complete — and nothing in a pipelined turn writes to the store's +// private-payload scratchpad, because the shim that writes to it is what is missing. // // One deliberate difference from `respond.ts`, shared with every other family on the // pipeline: an upstream that refused is answered in its own words, with the status it sent, -// rather than being quoted back inside an envelope this gateway wrote. And one that is this -// family's own: a stream that runs out before its terminal event still ends the run with -// `RESPONSES_MISSING_TERMINAL_MESSAGE`, but nothing turns that into the mid-stream `error` -// and `response.failed` pair `respond.ts` wrote for a client already being streamed to. +// rather than being quoted back inside an envelope this gateway wrote. import { analyzeResponsesAffinity } from './affinity/ingress.ts'; +import { wrapResponsesClientEgress } from './client-output.ts'; +import type { ResponsesServeFailure } from './errors.ts'; +import { hydrateResponsesPayload } from './items/hydrate.ts'; +import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve-prep.ts'; import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; @@ -61,23 +63,24 @@ import { } from '../interceptors.ts'; import { messagesWire } from '../messages/pipeline.ts'; import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; +import { tryCatchChatServeFailure } from '../shared/errors.ts'; import { createExternalImageLoader } from '../shared/external-image-loader.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; -import { wrapResponsesAffinityEgress } from './affinity/egress.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage, type Use } from '@floway-dev/pipeline'; -import { doneFrame, renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { doneFrame, eventFrame, renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, isResponsesTerminalEvent, responsesProtocolFrameToSSEFrame, RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, + type ClientResponseResource, + type ClientResponsesStreamEvent, type ResponsesStreamEvent, } from '@floway-dev/protocols/responses'; -import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { providerModelOf, toInternalDebugError, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateResponsesViaChatCompletions, translateResponsesViaMessages } from '@floway-dev/translate'; /** `/v1/responses` prefers its own wire, then the translated Messages path, then the @@ -86,6 +89,14 @@ export const responsesTarget = chatTargetPicker(['responses', 'messages', 'chat- /** What this family adds to the chat space. */ export interface ResponsesFacts extends ChatFacts { + /** Server-only state the gateway once attached to an item it emitted, keyed by that + * item's id, as the rows this turn hydrated carry it. It never reaches an upstream: it is + * what lets an item this turn re-emits be stored with the state it already had. + * + * Pairs rather than a `Map`, for the same reason the header keys are: a `Map` has no own + * properties, so it would be written into the dump as an empty object and the record would + * say the turn hydrated nothing. */ + 'request.chat.responses.privatePayloads': readonly (readonly [string, unknown])[]; /** What the client is actually sent — an object when it asked for one, SSE frames when it * asked to stream. The edge provides it, so a dump shows what the client received. */ 'response.chat.responses.rendered': Record | AsyncIterable; @@ -105,8 +116,12 @@ type R = { [P in K]: ResponsesFacts[P] }; * same frames that would have gone out are folded here instead, by the protocol's own * reassembly, which is what makes a stream that stopped short of its terminal event say so * rather than answer with half a response. + * + * The stored-items membrane's other half runs here too, and it takes the client's own + * payload rather than the record's, because by the fork the record holds the prepared one — + * `previous_response_id` expanded away — and the resource echoes what the client asked with. */ -const emitResponses = defineStage< +const emitResponses = (client: CanonicalResponsesPayload) => defineStage< R<'ingress.chat.responses.wantsStream'>, R<'ingress.chat.responses.wantsStream'>, R<'ingress.chat.responses.wantsStream' | 'response.chat.responses' | 'response.http.headers'>, @@ -153,23 +168,39 @@ const emitResponses = defineStage< }; } - // The turn's own state, written back into the frames the client is handed: a follow-up - // carrying it comes back to the upstream that issued it. This is the other half of the - // affinity the resolver read on the way down, and it has to sit here because it rewrites - // the frames — below the fold, and there would be nothing left to rewrite. - const frames = wrapResponsesAffinityEgress( + // Everything this protocol writes back into its own frames, in one place because every + // layer of it rewrites them and below the fold there would be nothing left to rewrite: + // the terminal restated from the items that actually closed, the turn's own state sealed + // into the carrier a follow-up comes back on, each complete item stored under its exact + // id beneath one response id this gateway minted, and the resource completed to what the + // schema requires of it. It runs before the fold rather than beside it, so a client that + // did not ask to stream is answered with the object the persisted frames add up to. + const frames = wrapResponsesClientEgress( answer.frames as AsyncIterable>, - affinityEgressOptions(use.gateway), + use.gateway, + client, ); if (!back['ingress.chat.responses.wantsStream']) { - return { - ...rest, - 'response.http.headers': forClient, - 'response.chat.responses.rendered': move( - await collectResponsesProtocolEventsToResult(frames) as unknown as Record, - ), - 'response.http.status': 200, - }; + try { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move( + await collectResponsesProtocolEventsToResult(frames) as unknown as Record, + ), + 'response.http.status': 200, + }; + } catch (error) { + // Nothing has gone out yet, so the fault is still a status. A turn the gateway could + // not finish — the snapshot the next turn would read, most often — is not one it can + // answer, and the client is told what broke rather than handed the half that arrived. + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move(internalErrorEnvelope(error)), + 'response.http.status': 502, + }; + } } return { ...rest, @@ -180,20 +211,87 @@ const emitResponses = defineStage< }, }); +/** What a fault the gateway is answerable for looks like on this protocol, with the stack + * that says where it happened: this is the gateway's own failure and not an upstream's, so + * the body is a diagnostic rather than something a client is meant to parse. */ +const internalErrorEnvelope = (error: unknown): Record => { + const debug = toInternalDebugError(error); + return { + error: { + type: debug.type, + name: debug.name, + message: debug.message, + stack: debug.stack, + cause: debug.cause, + target_api: debug.target_api, + }, + }; +}; + +/** The spec nests the `error` event's payload under `error`, and both official SDKs key + * their mid-stream throw on exactly that key; the same fields at the top level are yielded + * to them as an ordinary event instead. + * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L170-L177 + * https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/core/streaming.ts#L69-L71 + * https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/_streaming.py#L87-L98 */ +const streamErrorEvent = (error: unknown): ClientResponsesStreamEvent => { + const debug = toInternalDebugError(error); + return { + type: 'error', + error: { + message: debug.message, + code: debug.type, + name: debug.name, + stack: debug.stack, + cause: debug.cause, + target_api: debug.target_api, + }, + } as unknown as ClientResponsesStreamEvent; +}; + +/** "Any error incurred while streaming will be followed by a `response.failed` event." + * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L430 */ +const streamFailedEvent = (announced: ClientResponseResource, error: unknown): ClientResponsesStreamEvent => { + const debug = toInternalDebugError(error); + return { + type: 'response.failed', + response: { ...announced, status: 'failed', error: { code: debug.type, message: debug.message } }, + } as ClientResponsesStreamEvent; +}; + /** Every Responses event has an SSE form of its own, so the render is a straight map. What - * it adds is the terminator: the client's stream ends on the literal `[DONE]` payload - * whether or not the upstream's stream carried one, because that is what the transport - * reads to know the turn is over. - * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx?plain=1#L84 */ -const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ + * it adds is the two endings a client already being streamed to can be given. The ordinary + * one is the terminator: the client's stream ends on the literal `[DONE]` payload whether or + * not the upstream's stream carried one, because that is what the transport reads to know + * the turn is over. + * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx?plain=1#L84 + * + * The other is a break — an upstream that died mid-stream, a turn that could not be stored, + * a stream that ran out before saying how it ended. The status went out with the headers, so + * the failure has to be said in the protocol's own words, and it is said *instead of* the + * terminator: a stream that ended on `[DONE]` is a stream that finished. */ +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) { - // The upstream's own terminator is not the client's — the ending stops reading at the - // turn's terminal event, and one terminator is written below however the frames ended. - if (frame.type === 'done') continue; - yield responsesProtocolFrameToSSEFrame(frame); + // The last resource the client was shown, which is the one a `response.failed` restates: + // the id it names has to be the id the client already saw this turn under. + let announced: ClientResponseResource | undefined; + try { + for await (const frame of frames) { + // The upstream's own terminator is not the client's — the ending stops reading at the + // turn's terminal event, and one terminator is written below however the frames ended. + if (frame.type === 'done') continue; + if ('response' in frame.event) announced = frame.event.response; + yield responsesProtocolFrameToSSEFrame(frame); + } + yield responsesProtocolFrameToSSEFrame(doneFrame()); + } catch (error) { + yield sseFrame(JSON.stringify(streamErrorEvent(error)), 'error'); + // Nothing was announced when the break came before the first resource-bearing event, + // and there is no response to restate as failed. + if (announced !== undefined) { + yield responsesProtocolFrameToSSEFrame(eventFrame(streamFailedEvent(announced, error))); + } } - yield responsesProtocolFrameToSSEFrame(doneFrame()); })(), }); @@ -428,10 +526,16 @@ const billed = (identity: TelemetryModelIdentity, usage: BillableUsage | undefin /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is - * built from the request rather than being a constant. */ -const narrowing = (payload: CanonicalResponsesPayload): ChatNarrowing> => ({ + * built from the request rather than being a constant. + * + * It reads the request through a function because the one it has to read is the *prepared* + * one: a `previous_response_id` continuation carries the prior turn's state on items the + * client never sent, and a turn is pinned by what its items carry. The narrowing is built + * at assembly, before any fact exists, so the membrane hands the prepared payload across + * through the run's own cell rather than through the record. */ +const narrowing = (prepared: () => CanonicalResponsesPayload): ChatNarrowing> => ({ canServe: candidate => responsesTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeResponsesAffinity(payload, gateway.affinity.codec), + affinity: async gateway => await analyzeResponsesAffinity(prepared(), gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /responses endpoint.`, refuse: (status, message, reason) => ({ 'response.chat.responses.streamedUsage': null, @@ -452,6 +556,144 @@ const narrowing = (payload: CanonicalResponsesPayload): ChatNarrowing => ({ + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.chat.responses.streamedUsage': null, + 'response.chat.responses': { + status, + message, + envelope: { error: { message, type: 'invalid_request_error', ...extra } }, + }, +}); + +/** + * The stored-items membrane, on the way in. + * + * Three things, in an order the store fixes. A `previous_response_id` becomes the snapshot's + * items in front of this turn's input, and the id itself never reaches a wire — it names + * something only this gateway holds. Every item the turn now names is read back out of the + * store, so an item the client echoed by id is sent as the row we stored rather than as the + * client's copy of it, and whatever server-only state that row carried comes back with it. + * Then the items this turn *adds* are staged, so the snapshot the next turn continues from + * holds them alongside the history it inherited. + * + * It sits above the resolver because everything after it routes on the result: the payload + * affinity reads is the hydrated one, and a turn whose continuation does not resolve has + * nowhere to be routed to. Both of its refusals are answers it already holds, which is why + * it carries the `return` trait alongside `through`. + */ +const hydrateStoredItems = ( + client: CanonicalResponsesPayload, + prepared: (payload: CanonicalResponsesPayload) => void, +) => defineStage< + R<'request.chat.responses'>, + R<'request.chat.responses' | 'request.chat.responses.privatePayloads'>, + Record, + Record, + R<'response.chat.responses' | 'response.chat.responses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'hydrateStoredItems', + through: { + request: { + needs: ['request.chat.responses'], + consumes: [], + provides: ['request.chat.responses', 'request.chat.responses.privatePayloads'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { + provides: [ + 'response.chat.responses', + 'response.chat.responses.streamedUsage', + 'response.usage.billable', + 'response.http.headers', + ], + }, + execute: async (facts, next, use) => { + const store = use.gateway.store; + // The key holds what a client may send, whose `input` is a string or a list; this chain + // runs on the canonical form the entry normalized it to. + const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; + + let expanded: CanonicalResponsesPayload; + try { + expanded = await expandPreviousResponseId(asked, store); + } catch (error) { + if (!(error instanceof PreviousResponseNotFoundError)) throw error; + // OpenAI's own words for a continuation that does not resolve, byte for byte: Codex + // compares this body against upstream's. See the cross-references on + // `PreviousResponseNotFoundError`. + return move({ + ...facts, + ...refuseStoredItems(400, error.message, { param: 'previous_response_id', code: 'previous_response_not_found' }), + }); + } + + // Both lists: what the turn now names is read by id, and what the client itself sent is + // read by item hash as well, so a body repeated verbatim finds the row it already made. + await store.loadInputItems(expanded.input, client.input); + + let hydrated: ReturnType; + try { + hydrated = hydrateResponsesPayload(expanded, store); + } catch (error) { + const failure = tryCatchChatServeFailure(error); + if (failure?.kind !== 'item-not-found') throw error; + return move({ + ...facts, + ...refuseStoredItems(404, `Item with id '${failure.itemId}' not found.`, { param: 'input', code: null }), + }); + } + + // The client's own input, not the expansion's `item_reference` prefix: the prefix is + // already in the snapshot this turn inherited, and staging it again would repeat it. + await store.stageInputItems(client.input); + + prepared(hydrated.payload); + return await next({ + ...facts, + 'request.chat.responses': move(hydrated.payload), + 'request.chat.responses.privatePayloads': move([...hydrated.privatePayloads]), + }); + }, +}); + +/** + * Reseeds the store's per-attempt scratchpad from what the membrane hydrated. + * + * Below the fork because it is per attempt: what one attempt wrote into the scratchpad is + * not what the next one starts from, and re-running the suffix is what clears it. Nothing in + * a pipelined turn writes to it yet — the server-tool shim is what writes, and it is still + * an interceptor — so today this only puts back the state the stored rows already carried, + * which is what lets an item this turn re-emits be stored with it intact. + */ +const beginStoredAttempt = defineStage< + R<'request.chat.responses.privatePayloads'>, + R<'request.chat.responses.privatePayloads'>, + Record, + Record, + ChatServices +>({ + name: 'beginStoredAttempt', + through: { + request: { needs: ['request.chat.responses.privatePayloads'], consumes: [], provides: [] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => { + use.gateway.store.beginAttempt(new Map(facts['request.chat.responses.privatePayloads'])); + return await next(facts); + }, +}); + export type ResponsesServeEntry = R< 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.responses.wantsStream' | 'request.chat.responses' | 'serve.model' @@ -462,19 +704,26 @@ export type ResponsesServeExit = R< | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' >; -export const responsesServePipeline = (payload: CanonicalResponsesPayload): Pipeline => - compose('responsesServe', [ - emitResponses, +export const responsesServePipeline = (payload: CanonicalResponsesPayload): Pipeline => { + // One cell per run, written by the stage directly above the one that reads it. The + // resolver takes its narrowing at assembly, so this is where the prepared payload crosses + // from the membrane to the affinity walk; until the membrane has run, what the client sent + // is the whole of what is known about the turn. + let prepared = payload; + return compose('responsesServe', [ + emitResponses(payload), writeSettlement( handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), handedUp => (handedUp as { 'response.chat.responses.streamedUsage'?: unknown })['response.chat.responses.streamedUsage'] !== null, ), - resolveChatCandidates(narrowing(payload)), + hydrateStoredItems(payload, hydrated => { prepared = hydrated; }), + resolveChatCandidates(narrowing(() => prepared)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), owns: [], }), materializeAttempt('request.chat.responses'), + beginStoredAttempt, disableReasoningOnForcedToolChoiceForResponses, stripPromptCacheKeyForResponses, vendorDeepSeekNormalizeForResponses, @@ -487,3 +736,4 @@ export const responsesServePipeline = (payload: CanonicalResponsesPayload): Pipe wire: responsesWireFor, }), ]); +}; diff --git a/packages/gateway/src/data-plane/chat/responses/respond.ts b/packages/gateway/src/data-plane/chat/responses/respond.ts index 059b4d1cc2..6a74c1c70c 100644 --- a/packages/gateway/src/data-plane/chat/responses/respond.ts +++ b/packages/gateway/src/data-plane/chat/responses/respond.ts @@ -8,6 +8,7 @@ import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; import { settle } from '../../shared/telemetry/settle.ts'; import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; +import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; import { doneFrame, eventFrame, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; import { responsesProtocolFrameToSSEFrame, RESPONSES_MISSING_TERMINAL_MESSAGE, collectResponsesProtocolEventsToResult } from '@floway-dev/protocols/responses'; @@ -42,12 +43,14 @@ export const respondResponsesFailure = ( // Renders an upstream Responses result into the client HTTP/SSE response. An // events result drains to one JSON body (non-streaming) or is proxied frame by -// frame (streaming); anything else is a pre-stream failure. +// frame (streaming); anything else is a pre-stream failure. The egress it runs +// is the stored-items membrane's, so the context it takes is the chat one that +// carries the store and the affinity membrane. export const respondResponses = async ( c: Context, result: ExecuteResult> | PlainResult, wantsStream: boolean, - ctx: GatewayCtx, + ctx: ChatGatewayCtx, request: CanonicalResponsesPayload, ): Promise => { if (result.type !== 'events') return respondResponsesFailure(result, ctx); From 2ff1db6aeed71b0cb2d274b4874f823c7d4db75c Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 08:49:08 +0800 Subject: [PATCH 040/156] feat(chat): serve /v1/responses through its pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last chat family still on the old surface. Its chain was built, tested and complete, and nothing reached it: the route ran `responsesServe.generate` and `respondResponses`, so every `/v1/responses` turn was served by the onion the chain replaces. The entry is now a prologue and an epilogue around `responsesServePipeline`, and the store factory is the native one — this is the source that persists, and every other source gets the scratchpad `openChatPrologue` already hands out. Reading the body became a value rather than a throw, because a pipelined entry decides what to do with a body it could not read before it opens a run rather than after one unwound. What the reader rejected travels with the rejection: this protocol's envelope names the field at fault and the code for the condition, and `input` and `input[0]` are different sentences, as is a body with no `model`. A body that is not JSON at all is now a 400 rather than the 502 the throwing surface answered it with, which is what the three families routed before this one already do. `/v1/responses/compact` has not moved. It is a second operation over this protocol rather than another wire under its chain, so it stays on `responsesServe.compact` and keeps the entry's throw handling with it. Two turns the route can now serve get tests of their own, because the membrane they exercise is only reachable through it: a turn continued from the snapshot a `previous_response_id` names, and an `item_reference` no row answers. --- .../data-plane/chat/responses/http_test.ts | 79 +++++++++++++ .../src/data-plane/chat/responses/http.ts | 107 +++++++++++++++--- 2 files changed, 172 insertions(+), 14 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/http_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/http_test.ts index c38b686521..c39e52adfc 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/http_test.ts @@ -542,6 +542,85 @@ test('POST /v1/responses with an unresolvable previous_response_id renders the v assertEquals(body.error.code, 'previous_response_not_found'); }); +test('POST /v1/responses replays a stored turn when the next one names it as previous_response_id', async () => { + installRepo(); + const observedBodies: Array> = []; + const callResponses = vi.fn(async (_model, body): Promise => { + observedBodies.push(body as Omit); + return { + action: 'generate', ok: true, + events: makeProviderEvents(completedEvents(`resp_upstream_${observedBodies.length}`)), + modelKey: 'test-model-key', + headers: new Headers(), + }; + }); + const candidate = makeCandidate({ callResponses }); + queueResolution([candidate]); + queueResolution([candidate]); + const app = makeApp(); + + const first = await app.request('/v1/responses', { + method: 'POST', + headers: new Headers({ 'content-type': 'application/json' }), + body: JSON.stringify({ model: 'test-model', input: 'hello', store: true }), + }); + assertEquals(first.status, 200); + const stored = await first.json() as ResponsesResult; + + const second = await app.request('/v1/responses', { + method: 'POST', + headers: new Headers({ 'content-type': 'application/json' }), + body: JSON.stringify({ + model: 'test-model', + store: true, + previous_response_id: stored.id, + input: 'continue', + }), + }); + assertEquals(second.status, 200); + await second.json(); + + // The prior turn is replayed as the rows the store holds — this turn's own input last — + // and the snapshot id names something only this gateway has, so it never reaches the wire. + const continued = observedBodies[1]; + if (continued === undefined) throw new Error('expected a second upstream call'); + assertEquals(continued.previous_response_id, undefined); + assertEquals(continued.input, [ + { type: 'message', role: 'user', content: 'hello' }, + { + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'hi', annotations: [] }], + }, + { type: 'message', role: 'user', content: 'continue' }, + ]); +}); + +test('POST /v1/responses rejects an item_reference the store cannot resolve', async () => { + installRepo(); + + // No candidates need to be queued — hydration answers before routing runs. + const response = await makeApp().request('/v1/responses', { + method: 'POST', + headers: new Headers({ 'content-type': 'application/json' }), + body: JSON.stringify({ + model: 'test-model', + input: [{ type: 'item_reference', id: 'msg_missing' }], + }), + }); + + assertEquals(response.status, 404); + const body = await response.json() as { error: { message: string; type: string; param: string; code: string | null } }; + assertEquals(body.error, { + message: "Item with id 'msg_missing' not found.", + type: 'invalid_request_error', + param: 'input', + code: null, + }); +}); + test('POST /v1/responses and /v1/responses/compact reject a body without `model` with the OpenAI missing-parameter 400', async () => { installRepo(); diff --git a/packages/gateway/src/data-plane/chat/responses/http.ts b/packages/gateway/src/data-plane/chat/responses/http.ts index 6dd9321a89..c25a6a3f0e 100644 --- a/packages/gateway/src/data-plane/chat/responses/http.ts +++ b/packages/gateway/src/data-plane/chat/responses/http.ts @@ -1,16 +1,30 @@ +// POST /v1/responses, served through the pipeline, and POST /v1/responses/compact beside it +// — a second operation over this protocol rather than another wire under its chain, so it +// still goes through `responsesServe.compact`. +// +// The generate entry is a prologue and an epilogue around `responsesServePipeline`: read +// what the client sent, hand it over, and turn what the run answered with into a response. +// Everything between is stages, the stored-items membrane included — which is why a +// continuation that does not resolve is answered by the chain here and still caught as a +// throw on the compact route, which has not moved. + import { responsesInputErrorResult } from './errors.ts'; import { createResponsesHttpStore } from './items/store.ts'; +import { responsesServePipeline } from './pipeline.ts'; import { respondResponses, respondResponsesFailure } from './respond.ts'; import { PreviousResponseNotFoundError } from './serve-prep.ts'; import { responsesServe } from './serve.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; +import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../../shared/gateway-ctx.ts'; import { inboundHeaders } from '../../shared/inbound-headers.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; import { settle } from '../../shared/telemetry/settle.ts'; +import { openChatPrologue } from '../prologue.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; import { providerModelsUnavailableResponse } from '../shared/upstream-models-error.ts'; +import { move } from '@floway-dev/pipeline'; import type { CanonicalResponsesPayload, ResponsesRequestPayload } from '@floway-dev/protocols/responses'; import { internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; @@ -67,30 +81,95 @@ const respondToThrow = async (c: AuthedContext, error: unknown, requestBody: Req return await respondWithInternalError(c, error, requestBody, ctx); }; -const parsePayload = (requestBody: RequestBody): CanonicalResponsesPayload => - canonicalizeResponsesPayload(JSON.parse(new TextDecoder().decode(requestBody.bytes)) as ResponsesRequestPayload); +const parsePayload = (bytes: Uint8Array): CanonicalResponsesPayload => + canonicalizeResponsesPayload(JSON.parse(new TextDecoder().decode(bytes)) as ResponsesRequestPayload); + +/** The same read, as a value rather than a throw, because a pipelined entry decides what to + * do about a body it could not read before it opens a run rather than after one unwound. + * + * What the reader rejected travels with the rejection. This protocol's envelope names the + * field at fault and the code for the condition, and those are statements only the reader + * can make: `input` and `input[0]` are different sentences, and a body with no `model` is + * refused in words OpenAI's own clients already parse. */ +const readRequest = (bytes: Uint8Array): + | { type: 'ok'; payload: CanonicalResponsesPayload } + | { type: 'invalid'; message: string; param?: string; code?: string } => { + try { + return { type: 'ok', payload: parsePayload(bytes) }; + } catch (error) { + if (!(error instanceof TranslatorInputError)) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; + } + return { + type: 'invalid', + message: error.message, + ...(error.param === undefined ? {} : { param: error.param }), + ...(error.code === undefined ? {} : { code: error.code }), + }; + } +}; + +/** The rejection, as this protocol's clients read one. A reader that named neither field + * still fills both slots, because the envelope declares them and `input` is where a body + * this endpoint could not read went wrong. */ +const invalidRequestResponse = (invalid: { message: string; param?: string; code?: string }): Response => + Response.json( + { + error: { + message: invalid.message, + type: 'invalid_request_error', + param: invalid.param ?? 'input', + code: invalid.code ?? null, + }, + }, + { status: 400 }, + ); export const responsesHttp = { generate: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = parsePayload(requestBody); - const wantsStream = payload.stream === true; - ctx = createChatGatewayCtxFromHono(c, { wantsStream, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, (apiKey, requestStartedAt) => createResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined)); - const result = await responsesServe.generate({ payload, ctx, headers: inboundHeaders(c) }); - const response = await respondResponses(c, result, wantsStream, ctx, payload); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') { + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse(refused.gateway, invalidRequestResponse(request)); } + + const { payload } = request; + const wantsStream = payload.stream === true; + const prologue = openChatPrologue(c, ingress, { + wantsStream, + model: payload.model, + storeFactory: (apiKey, requestStartedAt) => createResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined), + }); + + return await serveThrough( + c, + prologue, + responsesServePipeline(payload), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.responses.wantsStream': wantsStream, + 'request.chat.responses': payload, + 'serve.model': payload.model, + }) as never, + facts => { + const rendered = facts['response.chat.responses.rendered']; + if (isFrames(rendered)) return { frames: rendered }; + return { body: JSON.stringify(rendered), contentType: 'application/json' }; + }, + facts => facts['response.chat.responses.streamedUsage'], + ); }, compact: async (c: AuthedContext): Promise => { const requestBody = await readRequestBody(c); let ctx: ChatGatewayCtx | undefined; try { - const payload = parsePayload(requestBody); + const payload = parsePayload(requestBody.bytes); ctx = createChatGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, (apiKey, requestStartedAt) => createResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined)); const result = await responsesServe.compact({ payload, ctx, headers: inboundHeaders(c) }); if (result.type === 'result') { From 92aea976e71351d011b9cf9b2853fe8338df5052 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 08:52:44 +0800 Subject: [PATCH 041/156] fix(responses): send an assistant item back in the content type the wire accepts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot's compaction translation and Azure-native compaction both emit assistant messages whose content blocks carry `type: 'input_text'`, and both then refuse those same items echoed back as input: `Invalid value: 'input_text'. Supported values are: 'output_text' and 'refusal'.` The old attempt boundary put the canonical type back on every payload it dialled with; the chain that replaced it did not, so a turn replaying prior upstream-produced history was refused. It was reachable before the route moved — the three families routed ahead of this one dial this wire too — and the membrane makes it reachable the way it actually bites: a snapshot the gateway itself stored and expanded back into the next turn. The rule speaks about this wire, so it sits in it, beside the role rewrite that states the same kind of thing. It is not needed on the translated ones: both translators read `input_text` and `output_text` the same way on assistant content, so a turn that leaves for Messages or Chat Completions never carried the disagreement. --- .../chat/responses-pipeline_test.ts | 29 ++++++++++++ .../src/data-plane/chat/responses/pipeline.ts | 46 +++++++++++++++++-- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index 507bf10e55..152b8a2796 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -205,6 +205,35 @@ describe('the responses chain', () => { expect(action).toBe('generate'); }); + // Copilot's compaction translation and Azure-native compaction both emit assistant messages + // whose content blocks say `input_text`, and both then refuse those same items echoed back + // as input. The wire puts the canonical type back on whatever reaches it, however the + // history got there — a client echo, or the snapshot the membrane replayed. + it('sends an assistant item back in the content type this wire accepts', async () => { + let sent: Record | undefined; + affinityPayload = { + ...payload, + input: [ + { type: 'message', role: 'assistant', content: [{ type: 'input_text', text: 'summary so far' }] }, + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'and then?' }] }, + ], + } as unknown as CanonicalResponsesPayload; + resolves([candidate(async (_model, body) => { + sent = body as Record; + return stream(completed('hi')); + })]); + + await serve(false); + + // Only the assistant's: `input_text` IS the correct type on a user message. + expect(sent).toMatchObject({ + input: [ + { role: 'assistant', content: [{ type: 'output_text', text: 'summary so far' }] }, + { role: 'user', content: [{ type: 'input_text', text: 'and then?' }] }, + ], + }); + }); + // Every event goes out under its own SSE name, and the client's stream ends on the literal // `[DONE]` the transport reads as the turn being over. it('writes the frames out when the client asked to stream', async () => { diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index 5dc50fd226..61745e5121 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -39,6 +39,7 @@ import { analyzeResponsesAffinity } from './affinity/ingress.ts'; import { wrapResponsesClientEgress } from './client-output.ts'; import type { ResponsesServeFailure } from './errors.ts'; import { hydrateResponsesPayload } from './items/hydrate.ts'; +import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve-prep.ts'; import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; @@ -412,17 +413,52 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< }, }); +/** + * What an upstream's Responses endpoint accepts on an assistant item it produced itself. + * Copilot's compaction translation and Azure-native compaction both emit assistant messages + * whose content blocks carry `type: 'input_text'`, and both then refuse those same items + * echoed back as input on the next turn. Every way prior upstream-produced history reaches a + * wire arrives here — a direct client echo, the snapshot the membrane expanded, a compaction + * tail — so this is the one place the canonical assistant content type is put back. + * + * Only this wire needs it. Both translators read `input_text` and `output_text` the same way + * on assistant content, so a turn that leaves for Messages or Chat Completions never carried + * the disagreement in the first place. + */ +const normalizeAssistantContentForResponses = defineStage< + R<'request.chat.responses'>, + R<'request.chat.responses'>, + Record, + Record, + ChatServices +>({ + name: 'normalizeAssistantContentForResponses', + through: { + request: { needs: ['request.chat.responses'], consumes: [], provides: ['request.chat.responses'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.chat.responses'] as CanonicalResponsesPayload; + const input = normalizeAssistantInputText(payload.input); + // A rewrite that changed nothing hands the same payload on, so the record shows no + // change where none happened. + if (input === payload.input) return await next(facts); + return await next({ ...facts, 'request.chat.responses': move({ ...payload, input }) }); + }, +}); + /** * The Responses wire, as the chain that dials it. * * Every source protocol that reaches an upstream over this endpoint runs this, whether the - * client spoke Responses or a handoff arrived here — which is what makes the two rules above - * the dial belong here. The role rewrite states what an upstream's Responses endpoint - * accepts; the cache-bucket fold speaks about the usage *this* wire reports and about the - * flag that describes it, and a translator emits the canonical form, which is the one case - * the fold has nothing to do with. + * client spoke Responses or a handoff arrived here — which is what makes the three rules + * above the dial belong here. The role rewrite and the assistant-content rewrite both state + * what an upstream's Responses endpoint accepts; the cache-bucket fold speaks about the usage + * *this* wire reports and about the flag that describes it, and a translator emits the + * canonical form, which is the one case the fold has nothing to do with. */ export const responsesWire = (streamedUsage: string): readonly Stage[] => [ + normalizeAssistantContentForResponses, applyRoleCompatibilityToResponses, normalizeExclusiveCachedTokensForResponses, callResponsesUpstream(streamedUsage), From 53c2a0f7bc12b3b4f2d2f9659f12e021dd8a349a Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 08:58:45 +0800 Subject: [PATCH 042/156] test(gateway): assemble the Responses chain alongside the other ten It was the one family whose pipeline is built from the request and had no caller in the assembly test, so a declaration that could never compose would have waited for a request to say so. --- packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 5848a013e1..82f0b762e1 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -11,6 +11,7 @@ import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/p import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; import { messagesServePipeline } from '../../../src/data-plane/chat/messages/pipeline.ts'; +import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; import { imagesServePipeline } from '../../../src/data-plane/images/pipeline.ts'; @@ -27,6 +28,7 @@ const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] ['chat completions', () => chatCompletionsServePipeline({ model: 'm', messages: [] } as never)], ['messages', () => messagesServePipeline({ model: 'm', messages: [] } as never)], ['gemini', () => geminiServePipeline({ model: 'm', contents: [] } as never)], + ['responses', () => responsesServePipeline({ model: 'm', input: [] } as never)], ]; describe('every family assembles', () => { From 000ae5655a10a21ccc57b8e50734f9adcd35d250 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 09:17:38 +0800 Subject: [PATCH 043/156] feat(messages): serve /v1/messages/count_tokens through its own chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Counting is a second operation over this protocol, not another wire under the generate pipeline, so it gets a chain of its own rather than a branch inside that one: an edge that writes one body, the candidate resolver, the fork, and an ending that measures and never opens a stream. What it measures is what generation would send. The three request rules already ported to stages run here in the order the interceptor chain ran them, and the web-search shim's request half runs above them as a stage of its own — a count taken on the client's own body would measure a request no upstream is ever sent, because the native server tool always reaches the wire as a client tool. Only a native Messages endpoint measures, so the chain dials directly instead of forking and the narrowing rejects a candidate that would serve generation over a translated wire. Measuring bills nothing and times nothing, which is what the replaced surface recorded, so there is no settlement stage. Three deliberate differences from the replaced entry, each shared with the generate entry beside it: a body that will not parse is a 400 in this protocol's own words rather than a 502, a dial that never connected fails over instead of ending the request, and the upstream's own forwardable headers reach the client. --- .../chat/messages-count-tokens_test.ts | 339 ++++++++++++++++++ .../data-plane/chat/messages/http_test.ts | 8 +- .../data-plane/chat/messages/count-tokens.ts | 279 ++++++++++++++ .../src/data-plane/chat/messages/http.ts | 97 ++--- .../src/data-plane/chat/messages/pipeline.ts | 6 +- 5 files changed, 665 insertions(+), 64 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts create mode 100644 packages/gateway/src/data-plane/chat/messages/count-tokens.ts diff --git a/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts new file mode 100644 index 0000000000..3d48a5c86e --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts @@ -0,0 +1,339 @@ +// The Messages count-tokens chain, run. What is written down here is what only running it +// can say: that a measurement is taken on the body generation would have sent — the payload +// affinity materialized, the alias' own rules, the three request rules and the web-search +// rewrite — that a refusal is answered in the upstream's own words after every candidate has +// been tried, and that measuring bills nothing. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { messagesCountTokensPipeline } from '../../../src/data-plane/chat/messages/count-tokens.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { AliasRules, ModelEndpoints } from '@floway-dev/protocols/common'; +import type { MessagesPayload } from '@floway-dev/protocols/messages'; +import { type FlagId, type MessagesUpstreamCallOptions, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +type CountTokens = ( + model: unknown, + body: unknown, + signal: AbortSignal | undefined, + opts: MessagesUpstreamCallOptions, +) => Promise; + +const candidate = ( + callMessagesCountTokens: CountTokens, + overrides: { + upstreamId?: string; + endpoints?: ModelEndpoints; + enabledFlags?: ReadonlySet; + rules?: AliasRules; + } = {}, +): ModelCandidate => { + const upstreamId = overrides.upstreamId ?? 'up_a'; + const endpoints = overrides.endpoints ?? { messages: {} }; + const enabledFlags = overrides.enabledFlags ?? new Set(); + return { + provider: { + upstreamId, kind: 'claude-code', name: upstreamId, + // Wide enough that a header reaching the provider proves the ending let it through, + // rather than proving only that the allowlist did. + inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callMessagesCountTokens: callMessagesCountTokens as never }), + }, + model: stubInternalModel( + { + id: 'claude-model', + endpoints, + providerModels: { [upstreamId]: stubProviderModel({ id: 'claude-model', endpoints, enabledFlags }) }, + }, + upstreamId, + ), + ...(overrides.rules === undefined ? {} : { rules: overrides.rules }), + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[], sawModel = true): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel, failedUpstreams: [] } as never); +}; + +const counted = (input: number): ProviderCallResult => ({ + response: Response.json({ input_tokens: input }), + modelKey: 'claude-model-key', +}); + +const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as MessagesPayload; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * payload the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: MessagesPayload = payload; + +const count = async ( + request: MessagesPayload = payload, + headers: readonly (readonly [string, string])[] = [], +) => { + const gateway = mockChatGatewayCtx({ wantsStream: false }); + return await run( + messagesCountTokensPipeline(request), + move({ + 'ingress.http.headers': headers, + 'ingress.chat.sourceProtocol': 'messages', + 'request.chat.messages': request, + 'serve.model': request.model, + }) as never, + { + gateway, + background: () => {}, + rememberCandidates: () => {}, + rememberChatSelection: () => {}, + chatPayloadFor: () => affinityPayload, + selectAffinity: () => { throw new Error('a measurement pins nothing; it must not select affinity'); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, + ); +}; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + affinityPayload = payload; + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the messages count-tokens chain', () => { + // The same statement the generate chain makes: what is measured is the body affinity + // materialized for this candidate, stamped with the id that candidate actually serves, + // with an alias' own rules overlaid on top of it. + it('measures the payload this attempt is owed, under the id the upstream resolved', async () => { + let sent: Record | undefined; + let seenModel: { readonly id: string } | undefined; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as MessagesPayload; + resolves([candidate(async (model, body) => { + seenModel = model as { readonly id: string }; + sent = body as Record; + return counted(11); + }, { rules: { reasoning: { effort: 'high' } } })]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.messages.rendered']).toEqual({ input_tokens: 11 }); + expect(seenModel?.id).toBe('claude-model'); + expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + // The id the client addressed does not travel: the provider re-stamps what it resolved. + expect(sent).not.toHaveProperty('model'); + // An alias' rules reach the measured body, so a count and the generation it precedes + // are taken on the same request. + expect(sent).toMatchObject({ output_config: { effort: 'high' } }); + }); + + // Counting observes the same gateway-level rewrites generation does, in the order the + // chain applies them, so a client is told what the request it is about to send costs. + it('applies the three request rules generation applies', async () => { + let sent: Record | undefined; + affinityPayload = { + ...payload, + system: 'x-anthropic-billing-header: token\ncch=deadbeef1234;\nbase rules', + messages: [ + { role: 'system', content: 'inline rules' }, + { role: 'user', content: 'hello' }, + ], + thinking: { type: 'enabled', budget_tokens: 1024 }, + output_config: { effort: 'high' }, + tool_choice: { type: 'tool', name: 'lookup' }, + } as unknown as MessagesPayload; + resolves([candidate(async (_model, body) => { sent = body as Record; return counted(9); }, { + enabledFlags: new Set([ + 'strip-billing-attribution', + 'disable-reasoning-on-forced-tool-choice', + 'rewrite-mid-conv-system-to-user', + ]), + })]); + + await count(); + + expect(sent).toEqual({ + max_tokens: 64, + system: 'base rules', + messages: [ + { role: 'user', content: 'inline rules' }, + { role: 'user', content: 'hello' }, + ], + thinking: { type: 'disabled' }, + tool_choice: { type: 'tool', name: 'lookup' }, + }); + }); + + // Anthropic's native server tool never reaches an upstream as itself — the shim sends an + // ordinary client tool — so a count taken on the client's own body would measure a request + // nobody is sent. + it('measures the client-tool shape the web-search shim would actually send', async () => { + let sent: MessagesPayload | undefined; + const asked = { ...payload, tools: [{ type: 'web_search_20260209', max_uses: 3 }] } as unknown as MessagesPayload; + affinityPayload = asked; + resolves([candidate(async (_model, body) => { sent = body as MessagesPayload; return counted(13); }, { + enabledFlags: new Set(['messages-web-search-shim']), + })]); + + await count(asked); + + const tool = sent?.tools?.[0] as { name?: string; type?: string; input_schema?: unknown } | undefined; + expect(tool?.name).toBe('web_search'); + expect(tool).not.toHaveProperty('type'); + expect(tool?.input_schema).toEqual({ + type: 'object', + properties: { query: { type: 'string', description: 'Search query' } }, + required: ['query'], + }); + }); + + // A tool definition this protocol rejects is an answer the chain already holds, so no + // upstream is asked what an unsendable request would cost. + it('refuses a native web-search definition the protocol rejects without dialling', async () => { + const asked = { + ...payload, + tools: [ + { type: 'web_search_20260209' }, + { type: 'web_search_20250305' }, + ], + } as unknown as MessagesPayload; + affinityPayload = asked; + let dialled = 0; + resolves([candidate(async () => { dialled += 1; return counted(0); }, { + enabledFlags: new Set(['messages-web-search-shim']), + })]); + + const { facts } = await count(asked); + + expect(dialled).toBe(0); + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.messages.rendered']).toMatchObject({ + type: 'error', + error: { type: 'invalid_request_error', message: 'Only one native web search tool definition is supported per request.' }, + }); + }); + + // The flag is the whole of the gate here: counting has only the native wire, so the + // structural half of the shim's condition — a translated target — can never hold. + it('leaves the native tool alone on a candidate the shim is off for', async () => { + let sent: MessagesPayload | undefined; + const asked = { ...payload, tools: [{ type: 'web_search_20260209', max_uses: 3 }] } as unknown as MessagesPayload; + affinityPayload = asked; + resolves([candidate(async (_model, body) => { sent = body as MessagesPayload; return counted(13); })]); + + await count(asked); + + expect(sent?.tools).toEqual([{ type: 'web_search_20260209', max_uses: 3 }]); + }); + + // Anthropic beta flags have a typed path of their own precisely so no provider's header + // allowlist can admit them, and a measurement asks the same question generation does. + it('hands the beta flags over on their own path and not as a header', async () => { + let seen: MessagesUpstreamCallOptions | undefined; + resolves([candidate(async (_model, _body, _signal, opts) => { seen = opts; return counted(5); })]); + + await count(payload, [ + ['anthropic-beta', 'context-1m-2025-08-07, advanced-tool-use-2025-11-20'], + ['x-trace', 'abc'], + ]); + + expect(seen?.anthropicBeta).toEqual(['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20']); + expect(seen?.headers.get('anthropic-beta')).toBeNull(); + expect(seen?.headers.get('x-trace')).toBe('abc'); + }); + + // A measurement one upstream refused is a measurement another may answer, which is what + // the fork is for — and the surviving refusal keeps the upstream's own status and words. + it('fails a refused measurement over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { + tried.push('busy'); + return { response: new Response('unavailable', { status: 503 }), modelKey: 'k' }; + }, { upstreamId: 'up_busy' }), + candidate(async () => { tried.push('free'); return counted(42); }, { upstreamId: 'up_free' }), + ]); + + const { facts } = await count(); + + expect(tried).toEqual(['busy', 'free']); + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.messages.rendered']).toEqual({ input_tokens: 42 }); + }); + + it('answers the last refusal in the upstream-s own status and words', async () => { + resolves([candidate(async () => ({ + response: new Response(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + modelKey: 'k', + }))]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(429); + expect(facts['response.chat.messages.rendered']).toEqual({ + type: 'error', + error: { type: 'rate_limit_error', message: 'slow down' }, + }); + }); + + // A refused connection is an outcome the fork has to be able to see, not a fault that ends + // the run. + it('fails a dial that never connected over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); }, { upstreamId: 'up_dead' }), + candidate(async () => { tried.push('alive'); return counted(7); }, { upstreamId: 'up_alive' }), + ]); + + const { facts } = await count(); + + expect(tried).toEqual(['dead', 'alive']); + expect(facts['response.chat.messages.rendered']).toEqual({ input_tokens: 7 }); + }); + + // Only a native Messages endpoint measures — no translation carries the question — so a + // candidate that would serve generation over a translated wire cannot serve this, and the + // refusal names the endpoint the client actually addressed. + it('refuses a candidate no wire can measure on, naming the count endpoint', async () => { + resolves([candidate(async () => counted(0), { endpoints: { chatCompletions: {} } })]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.messages.rendered']).toMatchObject({ + error: { + type: 'invalid_request_error', + message: 'Model claude-model does not support the /messages/count_tokens endpoint.', + }, + }); + }); + + // Measuring is not generating: no upstream charged for the answer, so the run has nothing + // to bill and says so rather than naming an entity that reported zero. + it('bills nothing for a measurement', async () => { + resolves([candidate(async () => counted(3))]); + + const { facts } = await count(); + + expect((facts as Record)['response.usage.billable']).toEqual([]); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/http_test.ts index d34606f0e2..fca20261f9 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages/http_test.ts @@ -220,7 +220,7 @@ test('POST /v1/messages rejects body anthropic_beta with a 400 before routing', }); test('POST /v1/messages/count_tokens proxies the upstream measurement body', async () => { - installRepo(); + const repo = installRepo(); const callMessagesCountTokens = vi.fn(async (): Promise => ({ response: new Response(JSON.stringify({ input_tokens: 99 }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), modelKey: 'k', @@ -237,6 +237,12 @@ test('POST /v1/messages/count_tokens proxies the upstream measurement body', asy const body = await response.json() as { input_tokens: number }; assertEquals(body.input_tokens, 99); assertEquals(callMessagesCountTokens.mock.calls.length, 1); + + // Measuring is not generating: the turn bills nothing and times nothing, so it leaves + // neither a usage row nor a performance sample behind. + await flushBackground(); + assertEquals(await repo.usage.listAll(), []); + assertEquals(await repo.performance.listAll(), []); }); test('POST /v1/messages forwards upstream response headers end-to-end (streaming) and strips hop-by-hop / cookies', async () => { diff --git a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts new file mode 100644 index 0000000000..da0fb0ee88 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts @@ -0,0 +1,279 @@ +// `/v1/messages/count_tokens` as a pipeline. +// +// emitMessagesTokenCount the edge: writes the measurement, or the refusal, as one body +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// materializeAttempt puts the payload this candidate is owed into the record +// the three request rules generation applies, and the web-search request preparation +// callMessagesCountTokensUpstream the ending: measures, and never opens a stream +// +// A second **operation** over this protocol rather than another wire under +// `messagesServePipeline`, which is why it is a chain of its own rather than a branch inside +// that one. It answers with a body and nothing else: no stream, and no billed turn — the +// replaced surface wrote neither a usage row nor a performance sample for a measurement, so +// there is no `writeSettlement` here and the ending states an empty billed set. +// +// There is one wire. Only a native Messages endpoint measures — no translation carries the +// question — so the chain dials directly instead of forking, and the rule that generation +// keeps inside its Messages wire (the system-role rewrite, which speaks about what that +// endpoint accepts) sits in the array here for the same reason it sits there. +// +// What it measures is what generation would send. The same three request rules run, in the +// order the interceptor chain ran them, and the web-search shim's request half runs above +// them: a client that asked for the native server tool is measured on the client-tool shape +// the gateway would actually put on the wire. + +import { analyzeMessagesAffinity } from './affinity/ingress.ts'; +import { renderMessagesError } from './errors.ts'; +import { prepareMessagesWebSearchShimRequest } from './interceptors/web-search-shim.ts'; +import type { MessagesFacts } from './pipeline.ts'; +import type { Failure } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { failover } from '../../pipeline/stages.ts'; +import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import { + applyRoleCompatibilityToMessages, + disableReasoningOnForcedToolChoiceForMessages, + stripBillingAttributionFromMessages, +} from '../interceptors.ts'; +import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { renderProtocolError } from '@floway-dev/protocols/common'; +import { parseAnthropicBetaHeader, type MessagesPayload } from '@floway-dev/protocols/messages'; +import { providerModelOf } from '@floway-dev/provider'; + +/** Counting has no translation path: only an upstream's own Messages endpoint can answer the + * question, so a candidate that would serve generation over a translated wire cannot serve + * this. */ +export const messagesCountTokensTarget = chatTargetPicker(['messages']); + +/** What a measurement hands up. It rides at this family's own response key — which is what + * lets the request rules generation runs be the same stages here — and the arm is narrowed + * because nothing in this chain opens a stream. */ +type TokenCount = { readonly kind: 'value'; readonly body: unknown } | Failure; + +type Counted = { [P in K]: TokenCount }; + +type M = { [P in K]: MessagesFacts[P] }; + +/** + * The outermost edge. A measurement is one object, so there is nothing to fold and nothing + * to stream: what this decides is only whether the client is shown the upstream's counts or + * the refusal that stands in for them. + */ +const emitMessagesTokenCount = defineStage< + Record, + Record, + Counted<'response.chat.messages'> & M<'response.http.headers'>, + M<'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers'>, + ChatServices +>({ + name: 'emitMessagesTokenCount', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.chat.messages', 'response.http.headers'], + consumes: ['response.chat.messages', 'response.http.headers'], + provides: ['response.chat.messages.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.chat.messages': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.messages.rendered': move(renderProtocolError(answer.body, () => renderMessagesError(answer.status, answer.message))), + 'response.http.status': answer.status, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.messages.rendered': move(answer.body as Record), + 'response.http.status': 200, + }; + }, +}); + +/** + * Puts the request into the shape the web-search shim would actually send. + * + * Anthropic's native `web_search_*` server tool becomes an ordinary client tool on the wire, + * and prior turns' results are rewritten back into the history the upstream issued them + * with — so a count taken on the client's own body would measure a request no upstream is + * ever sent. Only the rewrite runs here: there is no stream to intercept and no search to + * execute, because nothing is generated. + * + * `messages-web-search-shim` is the whole of the gate, where generation also engages the + * shim unconditionally on a translated wire: counting has only the native wire, so the + * structural half of that condition can never hold. + * + * A tool definition this protocol rejects is an answer it already holds, which is why it + * carries the `return` trait. + */ +const prepareMessagesWebSearchRequest = defineStage< + M<'request.chat.messages' | 'route.attempt'>, + M<'request.chat.messages'>, + Counted<'response.chat.messages'>, + Counted<'response.chat.messages'>, + Counted<'response.chat.messages'> & M<'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'prepareMessagesWebSearchRequest', + through: { + request: { + needs: ['request.chat.messages', 'route.attempt'], + consumes: [], + provides: ['request.chat.messages'], + }, + response: { needs: ['response.chat.messages'], consumes: [], provides: [] }, + }, + return: { provides: ['response.chat.messages', 'response.usage.billable', 'response.http.headers'] }, + execute: async (facts, next) => { + if (!facts['route.attempt'].flags.includes('messages-web-search-shim')) return await next(facts); + const prepared = prepareMessagesWebSearchShimRequest(facts['request.chat.messages']); + if (prepared.type === 'invalid-request') { + // Nothing was dialled, which is what an empty billed set and an empty header list say + // on the other two keys. + return move({ + ...facts, + 'response.chat.messages': { status: 400, message: prepared.message }, + 'response.usage.billable': [], + 'response.http.headers': [], + }); + } + // A request that engaged nothing comes back by identity, so the record shows no change + // where none happened. + if (prepared.payload === facts['request.chat.messages']) return await next(facts); + return await next({ ...facts, 'request.chat.messages': move(prepared.payload) }); + }, +}); + +/** + * The ending. It asks the upstream what the turn would cost and hands the answer up as a + * value — the one shape this operation ever produces, because a measurement is a body and + * never a stream. + */ +const callMessagesCountTokensUpstream = defineStage< + M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, + Counted<'response.chat.messages'> & M<'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'callMessagesCountTokensUpstream', + return: { + provides: ['response.chat.messages', 'response.usage.billable', 'response.http.headers'], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + + // The payload affinity materialized for this candidate, as every stage between the fork + // and here has rewritten it. The id the client addressed does not travel — the provider + // re-stamps whatever it resolved upstream — and an alias' own rules apply to the body + // that is sent, so what is measured is what generation would be charged for. + const payload = { ...facts['request.chat.messages'], model: candidate.model.id }; + if (candidate.rules !== undefined) applyRulesToUpstreamMessages(payload, candidate.rules); + const { model: _addressed, ...body } = payload; + + // Anthropic's beta flags have a typed path of their own, so no header allowlist can admit + // them, and they are the client's own only when the client spoke this protocol. + const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); + const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'messages' + ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) + : []; + headers.delete('anthropic-beta'); + + // Nothing here is billed: measuring is not generating, and an upstream that answered + // the question charged nothing for it. + const nothingBilled = { 'response.usage.billable': [] as const }; + + let response: Response; + try { + ({ response } = await candidate.provider.instance.callMessagesCountTokens( + providerModelOf(candidate), + body, + use.gateway.abortSignal, + { ...buildUpstreamCallOptions(candidate, use.gateway, headers), anthropicBeta }, + )); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so there are no headers to carry. + return move({ + ...facts, + ...nothingBilled, + 'response.chat.messages': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.http.headers': [], + }); + } + + const text = await response.text(); + let parsed: unknown; + try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } + + if (!response.ok) { + use.log.warn('upstream refused', { status: response.status }); + return move({ + ...facts, + ...nothingBilled, + 'response.chat.messages': { + status: response.status, + message: text, + ...(parsed === undefined ? {} : { body: parsed }), + }, + 'response.http.headers': [...response.headers], + }); + } + + return move({ + ...facts, + ...nothingBilled, + 'response.chat.messages': { kind: 'value' as const, body: parsed }, + 'response.http.headers': [...response.headers], + }); + }, +}); + +/** A candidate that cannot serve *this* request is not a candidate — and what the client's + * own turn carries decides the order the rest are tried in, which is why the narrowing is + * built from the request rather than being a constant. */ +const narrowing = (payload: MessagesPayload): ChatNarrowing> => ({ + canServe: candidate => messagesCountTokensTarget.canServe(candidate.model.endpoints), + affinity: async gateway => await analyzeMessagesAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the /messages/count_tokens endpoint.`, + refuse: (status, message) => ({ 'response.chat.messages': { status, message } }), + refuses: ['response.chat.messages'], +}); + +export type MessagesCountTokensEntry = M< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.messages' | 'serve.model' +>; + +export type MessagesCountTokensExit = M< + 'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers' +>; + +export const messagesCountTokensPipeline = (payload: MessagesPayload): Pipeline => + compose('messagesCountTokens', [ + emitMessagesTokenCount, + resolveChatCandidates(narrowing(payload)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages']), + owns: [], + }), + materializeAttempt('request.chat.messages'), + prepareMessagesWebSearchRequest, + stripBillingAttributionFromMessages, + disableReasoningOnForcedToolChoiceForMessages, + applyRoleCompatibilityToMessages, + callMessagesCountTokensUpstream, + ]); diff --git a/packages/gateway/src/data-plane/chat/messages/http.ts b/packages/gateway/src/data-plane/chat/messages/http.ts index ce1691781b..c73278e1c1 100644 --- a/packages/gateway/src/data-plane/chat/messages/http.ts +++ b/packages/gateway/src/data-plane/chat/messages/http.ts @@ -1,29 +1,21 @@ -// POST /v1/messages, served through the pipeline, and POST /v1/messages/count_tokens beside -// it — a second operation over this protocol rather than another wire under its chain, so it -// still goes through `messagesServe.countTokens`. +// POST /v1/messages and POST /v1/messages/count_tokens, both served through their own +// pipeline. Generation is one chain; counting is a second operation over the same protocol +// and is a chain of its own rather than another wire under the first. // -// The generate entry is a prologue and an epilogue around `messagesServePipeline`: read what -// the client sent, hand it over, and turn what the run answered with into a response. -// Everything between is stages. +// Each entry is a prologue and an epilogue around that chain: read what the client sent, +// hand it over, and turn what the run answered with into a response. Everything between is +// stages. -import { renderMessagesError, translatorInputErrorResult } from './errors.ts'; +import { messagesCountTokensPipeline } from './count-tokens.ts'; +import { renderMessagesError } from './errors.ts'; import { messagesKeepAlive, messagesServePipeline } from './pipeline.ts'; -import { respondMessages } from './respond.ts'; -import { messagesServe } from './serve.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; -import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; +import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; import { createNonResponsesSourceStore } from '../responses/items/store.ts'; -import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerModelsUnavailableResponse } from '../shared/upstream-models-error.ts'; import { move } from '@floway-dev/pipeline'; import type { MessagesPayload } from '@floway-dev/protocols/messages'; -import { internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; -import { TranslatorInputError } from '@floway-dev/translate'; // Reject `anthropic_beta` / `betas` in the body; the Messages protocol carries // them via the `anthropic-beta` HTTP header. @@ -47,33 +39,6 @@ const rejectBodyBetaResponse = (payload: MessagesPayload): Response | null => { ); }; -// Surfaces a pre-stream throw (malformed JSON body, an interceptor crash, -// etc.) as a Messages-shaped 502 with the same internal-error envelope the -// in-flow `internal-error` ExecuteResult produces. The caller passes its -// outer `ctx` when one was already constructed (so the dump row preserves -// the model attribution the request-time `requestedModel` stamped, and the -// throwing-candidate telemetry stamped in serve.ts survives onto the error -// row); a fresh ctx is minted only for pre-parse failures where no payload -// was available to read model from. -const respondWithInternalError = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - const verbatim = providerModelsUnavailableResponse(error); - if (verbatim !== null) return verbatim; - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const result = internalErrorResult(502, toInternalDebugError(error), effectiveCtx.attempt.telemetry); - const response = await respondMessages(c, result, false, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); -}; - -// Pre-stream caller-input failure raised by a translator → Messages-shaped -// 400 invalid_request_error envelope. Anything else falls through to the -// internal-error 502 path. -const respondToThrow = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - if (!(error instanceof TranslatorInputError)) return await respondWithInternalError(c, error, requestBody, ctx); - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const response = await respondMessages(c, translatorInputErrorResult(error, effectiveCtx.attempt.telemetry), false, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); -}; - /** A body the gateway could not read is reported in the words this protocol's own clients * parse, rather than as a fault of the gateway's. */ const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: MessagesPayload } | { type: 'invalid'; message: string } => { @@ -92,9 +57,6 @@ const refuse = (c: AuthedContext, ingress: Ingress, response: Response): Respons return finalizeGatewayResponse(refused.gateway, response); }; -const parsePayload = (requestBody: RequestBody): MessagesPayload => - JSON.parse(new TextDecoder().decode(requestBody.bytes)) as MessagesPayload; - export const messagesHttp = { generate: async (c: AuthedContext): Promise => { const ingress = await readIngress(c); @@ -137,19 +99,34 @@ export const messagesHttp = { }, countTokens: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = parsePayload(requestBody); - const rejected = rejectBodyBetaResponse(payload); - if (rejected) return rejected; - - ctx = createChatGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonResponsesSourceStore(apiKey.id)); - const result = await messagesServe.countTokens({ payload, ctx, headers: inboundHeaders(c) }); - const response = await respondMessages(c, result, false, ctx); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') { + return refuse(c, ingress, Response.json(renderMessagesError(400, request.message), { status: 400 })); } + + const { payload } = request; + const rejected = rejectBodyBetaResponse(payload); + if (rejected !== null) return refuse(c, ingress, rejected); + + const prologue = openChatPrologue(c, ingress, { + wantsStream: false, + model: payload.model, + storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + messagesCountTokensPipeline(payload), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'messages', + 'request.chat.messages': payload, + 'serve.model': payload.model, + }) as never, + // A measurement is one body however the turn went, so there is never a stream to write. + facts => ({ body: JSON.stringify(facts['response.chat.messages.rendered']), contentType: 'application/json' }), + ); }, }; diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index 288b657354..a1ed26e6b6 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -12,9 +12,9 @@ // handoff followed by that protocol's own wire. The stage above cannot tell which ran. // // `/v1/messages/count_tokens` is not one of them: it is a second operation over this -// protocol rather than another wire under this pipeline, and it is its own step. This -// family's own interceptors are not all stages yet either, so the array between the -// materialized payload and the fork is short rather than complete. +// protocol rather than another wire under this pipeline, so it is a chain of its own in +// `count-tokens.ts`. This family's own interceptors are not all stages yet either, so the +// array between the materialized payload and the fork is short rather than complete. import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { renderMessagesError } from './errors.ts'; From e7fc96276fbf674651ecf16c8bdaf5acdb1f0a78 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 09:26:12 +0800 Subject: [PATCH 044/156] feat(gemini): serve :countTokens through its own chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Counting is a second operation over this protocol, not another wire under the generate pipeline. Gemini has no endpoint that answers it, and unlike generation there is nothing to pick between — no protocol but Messages measures — so the translation sits in the array rather than under a fork. The pair is not a handoff: a handoff maps frames and there are none here. It consumes this protocol's request key, provides Messages', and on the way back reads Anthropic's `input_tokens` (or a translated upstream's `total_tokens`) out as Google's `{ totalTokens }`. A body carrying neither is the gateway failing to read the upstream, so it is a 502 the fork can move past. The three protocol-shape strippers run for the reason they run on generation: what no translation can carry cannot be sent. The thought suppressor does not — it rewrites a stream, and a measurement has none. Below the pair the chain is the Messages count-tokens wire, extracted in the commit before this one, so a Gemini measurement and a Messages one reach the upstream through the same rules. Two deliberate differences from the replaced entry, both shared with the generate entry beside it: a body that will not parse is a 400 in this protocol's own words rather than a 500 internal-error envelope, and an upstream that refused with a body of its own is answered in that body rather than having it quoted inside a Google-RPC envelope. --- .../chat/gemini-count-tokens_test.ts | 225 ++++++++++++++++++ .../data-plane/chat/gemini/count-tokens.ts | 200 ++++++++++++++++ .../src/data-plane/chat/gemini/http.ts | 136 ++++------- .../src/data-plane/chat/gemini/pipeline.ts | 6 +- .../data-plane/chat/messages/count-tokens.ts | 45 ++-- 5 files changed, 507 insertions(+), 105 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts create mode 100644 packages/gateway/src/data-plane/chat/gemini/count-tokens.ts diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts new file mode 100644 index 0000000000..a4eb9bd098 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts @@ -0,0 +1,225 @@ +// The Gemini count-tokens chain, run. What is written down here is what only running it can +// say: that the question is asked in Messages and the answer read back out as Google's, that +// what no translation can carry is gone before the question is asked, that a refusal keeps +// the upstream's own status, and that measuring bills nothing. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { geminiCountTokensPipeline } from '../../../src/data-plane/chat/gemini/count-tokens.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { ModelEndpoints } from '@floway-dev/protocols/common'; +import type { GeminiPayload } from '@floway-dev/protocols/gemini'; +import type { MessagesPayload } from '@floway-dev/protocols/messages'; +import type { MessagesUpstreamCallOptions, ModelCandidate, ProviderCallResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +type CountTokens = ( + model: unknown, + body: unknown, + signal: AbortSignal | undefined, + opts: MessagesUpstreamCallOptions, +) => Promise; + +const candidate = ( + callMessagesCountTokens: CountTokens, + overrides: { upstreamId?: string; endpoints?: ModelEndpoints } = {}, +): ModelCandidate => { + const upstreamId = overrides.upstreamId ?? 'up_a'; + const endpoints = overrides.endpoints ?? { messages: {} }; + return { + provider: { + upstreamId, kind: 'claude-code', name: upstreamId, + // Wide enough that a header reaching the provider proves the ending let it through, + // rather than proving only that the allowlist did. + inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callMessagesCountTokens: callMessagesCountTokens as never }), + }, + model: stubInternalModel( + { + id: 'claude-model', + endpoints, + limits: { max_output_tokens: 4096 }, + providerModels: { [upstreamId]: stubProviderModel({ id: 'claude-model', endpoints }) }, + }, + upstreamId, + ), + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const answered = (body: unknown): ProviderCallResult => ({ + response: Response.json(body), + modelKey: 'claude-model-key', +}); + +const payload: GeminiPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; + +const count = async ( + request: GeminiPayload = payload, + headers: readonly (readonly [string, string])[] = [], +) => { + const gateway = mockChatGatewayCtx({ wantsStream: false }); + return await run( + geminiCountTokensPipeline(request), + move({ + 'ingress.http.headers': headers, + 'ingress.chat.sourceProtocol': 'gemini', + 'request.chat.gemini': request, + 'serve.model': 'gemini-model', + }) as never, + { + gateway, + background: () => {}, + rememberCandidates: () => {}, + rememberChatSelection: () => {}, + chatPayloadFor: () => request, + selectAffinity: () => { throw new Error('a measurement pins nothing; it must not select affinity'); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, + ); +}; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the gemini count-tokens chain', () => { + // Gemini has no endpoint that answers this, so the question is asked in Messages and the + // counts are read back out under the name this protocol gives them. + it('asks the question in Messages and answers in Google-s own envelope', async () => { + let sent: MessagesPayload | undefined; + let seenModel: { readonly id: string } | undefined; + resolves([candidate(async (model, body) => { + seenModel = model as { readonly id: string }; + sent = body as MessagesPayload; + return answered({ input_tokens: 42 }); + })]); + + const { facts } = await count({ ...payload, systemInstruction: { parts: [{ text: 'system' }] } }); + + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.gemini.rendered']).toEqual({ totalTokens: 42 }); + expect(seenModel?.id).toBe('claude-model'); + expect(sent?.system).toBeDefined(); + expect(sent?.messages).toMatchObject([{ role: 'user', content: [{ type: 'text', text: 'hi' }] }]); + // `stream` says how an answer would be delivered, and a measurement delivers none. + expect(sent).not.toHaveProperty('stream'); + }); + + // Anthropic's own endpoint answers `input_tokens`; a Copilot upstream's translated count + // answers `total_tokens`. Either is the number this protocol calls `totalTokens`. + it('reads the other dialect an upstream states its counts in', async () => { + resolves([candidate(async () => answered({ total_tokens: 19 }))]); + + const { facts } = await count(); + + expect(facts['response.chat.gemini.rendered']).toEqual({ totalTokens: 19 }); + }); + + // A body carrying neither figure is an answer this gateway cannot read, which is the + // gateway failing to serve — and it is a failure the fork can move past, so another + // upstream gets asked before the client is told. + it('fails an unreadable count over, then reports it as the gateway-s own', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('odd'); return answered({ unexpected: true }); }, { upstreamId: 'up_odd' }), + candidate(async () => { tried.push('odder'); return answered({ unexpected: true }); }, { upstreamId: 'up_odder' }), + ]); + + const { facts } = await count(); + + expect(tried).toEqual(['odd', 'odder']); + expect(facts['response.http.status']).toBe(502); + expect(facts['response.chat.gemini.rendered']).toEqual({ + error: { code: 502, message: 'Invalid upstream token counting response.', status: 'UNAVAILABLE' }, + }); + }); + + // Gemini has no wire of its own, so what no translation can carry cannot be sent whichever + // candidate answers — and a count taken before the strip would measure a request the + // translator would refuse. + it('strips what no translation carries before asking', async () => { + let sent: MessagesPayload | undefined; + resolves([candidate(async (_model, body) => { sent = body as MessagesPayload; return answered({ input_tokens: 5 }); })]); + + const { facts } = await count({ + contents: [{ role: 'user', parts: [{ text: 'hi' }, { fileData: { fileUri: 'gs://x', mimeType: 'text/plain' } }] }], + tools: [{ googleSearch: {} }], + safetySettings: [{ category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_NONE' }], + } as unknown as GeminiPayload); + + expect(facts['response.chat.gemini.rendered']).toEqual({ totalTokens: 5 }); + // A tool group left declaring no function is not a tool group, so `tools` goes with it. + expect(sent).not.toHaveProperty('tools'); + expect(JSON.stringify(sent)).not.toContain('fileData'); + expect(sent).not.toHaveProperty('safetySettings'); + }); + + // Anthropic's beta flags are the client's own only when the client spoke that protocol, and + // a Gemini turn asked for nothing on that wire. + it('does not carry another protocol-s beta flags across the boundary', async () => { + let seen: MessagesUpstreamCallOptions | undefined; + resolves([candidate(async (_model, _body, _signal, opts) => { seen = opts; return answered({ input_tokens: 1 }); })]); + + await count(payload, [['anthropic-beta', 'must-not-cross-source-protocols'], ['x-trace', 'abc']]); + + expect(seen?.anthropicBeta).toEqual([]); + expect(seen?.headers.get('anthropic-beta')).toBeNull(); + expect(seen?.headers.get('x-trace')).toBe('abc'); + }); + + it('keeps the status of an upstream that refused, and names a refusal that said nothing', async () => { + resolves([candidate(async () => ({ response: new Response('', { status: 503 }), modelKey: 'k' }))]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(503); + expect(facts['response.chat.gemini.rendered']).toEqual({ + error: { code: 503, message: 'Upstream token counting request failed.', status: 'UNAVAILABLE' }, + }); + }); + + // Only an upstream's own Messages endpoint measures, so a candidate that would serve + // generation over a translated wire cannot serve this. + it('refuses a candidate no wire can measure on, naming the action', async () => { + resolves([candidate(async () => answered({ input_tokens: 0 }), { endpoints: { chatCompletions: {} } })]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.gemini.rendered']).toEqual({ + error: { code: 400, message: 'Model gemini-model does not support countTokens.', status: 'INVALID_ARGUMENT' }, + }); + }); + + it('bills nothing for a measurement', async () => { + resolves([candidate(async () => answered({ input_tokens: 3 }))]); + + const { facts } = await count(); + + expect((facts as Record)['response.usage.billable']).toEqual([]); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts new file mode 100644 index 0000000000..8ff80bede0 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts @@ -0,0 +1,200 @@ +// Gemini's `:countTokens` as a pipeline. +// +// emitGeminiTokenCount the edge: writes the measurement, or the refusal, as one body +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// materializeAttempt puts the payload this candidate is owed into the record +// the three strippers the same protocol-shape cleanups generation applies +// measureGeminiAsMessages the pair: asks the question in Messages and reads the answer back +// the Messages count wire the ending, shared with `/v1/messages/count_tokens` +// +// A second **operation** over this protocol rather than another wire under +// `geminiServePipeline`, so it is a chain of its own. Like generation it has no wire of its +// own — the provider surface has no Gemini call — but unlike generation it has only one +// target: no protocol but Messages answers "what would this cost", so there is nothing to +// pick between and the pair is in the array rather than under a fork. +// +// The pair is not `handOff`. A handoff maps frames, and there are none here: what crosses the +// protocol boundary is one measurement, and reading Anthropic's counts back out as Google's +// `{ totalTokens }` is the whole of the mapping. +// +// The strippers run for the reason they run on generation: what no translation can carry +// cannot be sent, so it goes at source. The thought suppressor does not — it rewrites a +// stream, and a measurement has none. + +import { analyzeGeminiAffinity } from './affinity/ingress.ts'; +import { renderGeminiError } from './errors.ts'; +import type { GeminiFacts } from './pipeline.ts'; +import { asJsonObject, readJsonNumber } from '../../../shared/json-helpers.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { failover } from '../../pipeline/stages.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import { + stripSafetySettingsFromGemini, + stripUnsupportedPartFieldsFromGemini, + stripUnsupportedToolsFromGemini, +} from '../interceptors.ts'; +import { messagesCountTokensWire, type TokenCountAnswer } from '../messages/count-tokens.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { renderProtocolError } from '@floway-dev/protocols/common'; +import type { GeminiPayload } from '@floway-dev/protocols/gemini'; +import { translateGeminiViaMessages } from '@floway-dev/translate'; + +/** Counting is reachable only over an upstream's own Messages endpoint: no other protocol + * answers the question, and no translation invents an answer. */ +export const geminiCountTokensTarget = chatTargetPicker(['messages']); + +type Counted = { [P in K]: TokenCountAnswer }; + +type G = { [P in K]: GeminiFacts[P] }; + +/** + * The outermost edge. A measurement is one object, so there is nothing to fold and nothing to + * stream: what this decides is only whether the client is shown the count or the refusal that + * stands in for it. + */ +const emitGeminiTokenCount = defineStage< + Record, + Record, + Counted<'response.chat.gemini'> & G<'response.http.headers'>, + G<'response.chat.gemini.rendered' | 'response.http.status' | 'response.http.headers'>, + ChatServices +>({ + name: 'emitGeminiTokenCount', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.chat.gemini', 'response.http.headers'], + consumes: ['response.chat.gemini', 'response.http.headers'], + provides: ['response.chat.gemini.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.chat.gemini': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.gemini.rendered': move(renderProtocolError(answer.body, () => renderGeminiError(answer.status, answer.message))), + 'response.http.status': answer.status, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.gemini.rendered': move(answer.body as Record), + 'response.http.status': 200, + }; + }, +}); + +/** + * Asks the question in Messages, and reads the answer back out as Google's. + * + * It consumes this protocol's request key and provides the target's, exactly as a handoff + * does, so the wire below sees only Messages and cannot tell it was reached by translation. + * What differs is the way back: there are no frames to map, only counts. + */ +const measureGeminiAsMessages = defineStage< + G<'request.chat.gemini' | 'route.attempt'>, + G<'request.chat.messages'>, + Counted<'response.chat.messages'>, + Counted<'response.chat.gemini'>, + ChatServices +>({ + name: 'measureGeminiAsMessages', + through: { + request: { + needs: ['request.chat.gemini', 'route.attempt'], + consumes: ['request.chat.gemini'], + provides: ['request.chat.messages'], + }, + response: { + needs: ['response.chat.messages'], + consumes: ['response.chat.messages'], + provides: ['response.chat.gemini'], + }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const trip = await translateGeminiViaMessages(facts['request.chat.gemini'], { + model: candidate.model.id, + fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, + }); + // The translator writes a turn, and a turn says how its answer is delivered. There is no + // answer to deliver here, so the field does not travel. + const { stream: _stream, ...target } = trip.target; + + const { 'request.chat.gemini': _asked, ...down } = facts; + const back = await next({ ...down, 'request.chat.messages': move(target) }); + const { 'response.chat.messages': counted, ...up } = back; + return { ...up, 'response.chat.gemini': move(asGeminiTokenCount(counted)) }; + }, +}); + +/** + * Anthropic's counts, as Google states them. + * + * The upstream body is provider-specific: Anthropic's own endpoint answers `input_tokens`, + * and a Copilot upstream's translated count answers `total_tokens`. Either is the number this + * protocol calls `totalTokens`; a body carrying neither is an answer this gateway cannot + * read, which is the gateway failing to serve rather than anything the client can fix. + */ +const asGeminiTokenCount = (counted: TokenCountAnswer): TokenCountAnswer => { + if (isFailure(counted)) { + return { + ...counted, + // An upstream that refused with an empty body left nothing to quote, and a refusal that + // says nothing at all is not one a caller can act on. + message: counted.message === '' ? 'Upstream token counting request failed.' : counted.message, + }; + } + const body = asJsonObject(counted.body); + const totalTokens = readJsonNumber(body?.input_tokens) ?? readJsonNumber(body?.total_tokens); + if (totalTokens === null) return { status: 502, message: 'Invalid upstream token counting response.' }; + return { kind: 'value', body: { totalTokens } }; +}; + +/** A candidate that cannot serve *this* request is not a candidate — and what the client's + * own turn carries decides the order the rest are tried in, which is why the narrowing is + * built from the request rather than being a constant. */ +const narrowing = (payload: GeminiPayload): ChatNarrowing> => ({ + canServe: candidate => geminiCountTokensTarget.canServe(candidate.model.endpoints), + affinity: async gateway => await analyzeGeminiAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support countTokens.`, + refuse: (status, message) => ({ 'response.chat.gemini': { status, message } }), + refuses: ['response.chat.gemini'], +}); + +export type GeminiCountTokensEntry = G< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.gemini' | 'serve.model' +>; + +export type GeminiCountTokensExit = G< + 'response.chat.gemini.rendered' | 'response.http.status' | 'response.http.headers' +>; + +export const geminiCountTokensPipeline = (payload: GeminiPayload): Pipeline => + compose('geminiCountTokens', [ + emitGeminiTokenCount, + resolveChatCandidates(narrowing(payload)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini']), + owns: [], + }), + materializeAttempt('request.chat.gemini'), + stripUnsupportedPartFieldsFromGemini, + stripUnsupportedToolsFromGemini, + stripSafetySettingsFromGemini, + measureGeminiAsMessages, + ...messagesCountTokensWire, + ]); diff --git a/packages/gateway/src/data-plane/chat/gemini/http.ts b/packages/gateway/src/data-plane/chat/gemini/http.ts index abd948c95c..d587fe5c62 100644 --- a/packages/gateway/src/data-plane/chat/gemini/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini/http.ts @@ -1,29 +1,22 @@ -// `/v1beta/models/:modelAction`, whose `:generateContent` and `:streamGenerateContent` -// actions are served through the pipeline. `:countTokens` is a second operation over this -// protocol rather than another wire under its chain, so it still goes through -// `geminiServe.countTokens`. +// `/v1beta/models/:modelAction`, whose three actions are each served through a pipeline. +// `:generateContent` and `:streamGenerateContent` are one chain; `:countTokens` is a second +// operation over the same protocol and is a chain of its own rather than another wire under +// the first. // -// The generate entry is a prologue and an epilogue around `geminiServePipeline`: read what -// the client sent, hand it over, and turn what the run answered with into a response. -// Everything between is stages. +// Each entry is a prologue and an epilogue around that chain: read what the client sent, hand +// it over, and turn what the run answered with into a response. Everything between is stages. -import { renderGeminiError, translatorInputErrorResult } from './errors.ts'; +import { geminiCountTokensPipeline } from './count-tokens.ts'; +import { renderGeminiError } from './errors.ts'; import { geminiServePipeline } from './pipeline.ts'; -import { geminiInternalRpcErrorResponse, geminiRpcErrorResponse, respondGemini } from './respond.ts'; -import { geminiServe } from './serve.ts'; +import { geminiRpcErrorResponse } from './respond.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; -import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; -import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; +import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; import { openChatPrologue } from '../prologue.ts'; import { createNonResponsesSourceStore } from '../responses/items/store.ts'; -import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; import { move } from '@floway-dev/pipeline'; import type { GeminiContent, GeminiPayload } from '@floway-dev/protocols/gemini'; -import { internalErrorResult, ProviderModelsUnavailableError, toInternalDebugError } from '@floway-dev/provider'; -import { TranslatorInputError } from '@floway-dev/translate'; interface GeminiModelAction { readonly model: string; @@ -51,53 +44,6 @@ const parseGeminiCountTokensPayload = (body: unknown): GeminiPayload => { return shape.generateContentRequest ?? { contents: shape.contents }; }; -const parseGeminiBodyBytes = (requestBody: RequestBody, project: (body: unknown) => T): T | Response => { - try { - const raw = JSON.parse(new TextDecoder().decode(requestBody.bytes)) as unknown; - return project(raw); - } catch (error) { - return geminiInternalRpcErrorResponse(500, error); - } -}; - -// Surfaces a pre-stream throw as a Gemini-RPC envelope, routing through -// `respondGemini` so the dump records the failure exactly as the sibling -// HTTP handlers do. `TranslatorInputError` renders a 400 INVALID_ARGUMENT -// envelope (caller-input violation). A `ProviderModelsUnavailableError` -// carrying an upstream HTTP body relays that body through the `api-error` -// path with `source: 'upstream'`; everything else collapses to an -// `internal-error` result rendered as the Gemini internal-error envelope -// (status, code, message, stack, cause, target_api). The throwing- -// candidate telemetry stamped in serve.ts survives onto the error row via -// `ctx.attempt.telemetry` so a mid-attempt throw still lands in -// performance_summary against the throwing upstream. -const respondWithGeminiError = async ( - c: AuthedContext, - error: unknown, - ctx: ChatGatewayCtx, - wantsStream: boolean, -): Promise => { - if (error instanceof TranslatorInputError) { - const response = await respondGemini(c, translatorInputErrorResult(error, ctx.attempt.telemetry), wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); - } - if (error instanceof ProviderModelsUnavailableError && error.httpResponse) { - const { status, headers, body } = error.httpResponse; - const apiErrorResult = { - type: 'api-error' as const, - source: 'upstream' as const, - status, - headers: new Headers(headers), - body: new TextEncoder().encode(body), - }; - const response = await respondGemini(c, apiErrorResult, wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); - } - const internalResult = internalErrorResult(500, toInternalDebugError(error), ctx.attempt.telemetry); - const response = await respondGemini(c, internalResult, wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); -}; - // Single entry for `/v1beta/models/:modelAction`. Splits the model and action // once, then dispatches to the matching sub-handler. Keeping the parse here // means the sub-handlers see a validated `(model, action)` pair and never @@ -113,28 +59,28 @@ export const geminiHttp = async (c: AuthedContext): Promise => { }; /** A body the gateway could not read is reported in the words this protocol's own clients - * parse, rather than as a fault of the gateway's. */ -const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: GeminiPayload } | { type: 'invalid'; message: string } => { + * parse, rather than as a fault of the gateway's. Each action reads the same bytes into its + * own shape, which is why the projection is the caller's. */ +const readRequest = (bytes: Uint8Array, project: (body: unknown) => T): { type: 'ok'; payload: T } | { type: 'invalid'; message: string } => { try { - return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as GeminiPayload }; + return { type: 'ok', payload: project(JSON.parse(new TextDecoder().decode(bytes)) as unknown) }; } catch (error) { return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } }; +/** A request the gateway refused before it reached a pipeline: there is no model to resolve + * and no attempt to make, so there is nothing for a run to record beyond the refusal. */ +const refuse = (c: AuthedContext, ingress: Ingress, message: string): Response => { + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse(refused.gateway, Response.json(renderGeminiError(400, message), { status: 400 })); +}; + const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: boolean): Promise => { const ingress = await readIngress(c); - const request = readRequest(ingress.body.bytes); - if (request.type === 'invalid') { - // A request the gateway could not read never reaches a pipeline: there is no model to - // resolve and no attempt to make, so there is nothing for a run to record. - const refused = openPrologue(c, ingress, { wantsStream: false }); - refused.gateway.dump?.error('gateway'); - return finalizeGatewayResponse( - refused.gateway, - Response.json(renderGeminiError(400, request.message), { status: 400 }), - ); - } + const request = readRequest(ingress.body.bytes, (body): GeminiPayload => body as GeminiPayload); + if (request.type === 'invalid') return refuse(c, ingress, request.message); const { payload } = request; const prologue = openChatPrologue(c, ingress, { @@ -166,16 +112,28 @@ const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: b }; const runGeminiCountTokens = async (c: AuthedContext, model: string): Promise => { - const requestBody = await readRequestBody(c); - const payload = parseGeminiBodyBytes(requestBody, parseGeminiCountTokensPayload); - if (payload instanceof Response) return payload; + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes, parseGeminiCountTokensPayload); + if (request.type === 'invalid') return refuse(c, ingress, request.message); - const ctx = createChatGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonResponsesSourceStore(apiKey.id)); - try { - const result = await geminiServe.countTokens({ payload, ctx, model, headers: inboundHeaders(c) }); - const response = await respondGemini(c, result, false, ctx); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondWithGeminiError(c, error, ctx, false); - } + const { payload } = request; + const prologue = openChatPrologue(c, ingress, { + wantsStream: false, + model, + storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + geminiCountTokensPipeline(payload), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'gemini', + 'request.chat.gemini': payload, + 'serve.model': model, + }) as never, + // A measurement is one body however the turn went, so there is never a stream to write. + facts => ({ body: JSON.stringify(facts['response.chat.gemini.rendered']), contentType: 'application/json' }), + ); }; diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 2082168aec..60e3ce3b1b 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -25,9 +25,11 @@ // `requireGeminiTerminal` is for — a Chat Completions stream that closed cleanly without ever // reporting a finish reason translates into candidates that never finish. // +// `:countTokens` is not one of the three: it is a second operation over this protocol rather +// than another wire under this pipeline — no stream, no billable turn, and a +// `{ totalTokens }` envelope of its own — so it is a chain of its own in `count-tokens.ts`. +// // What is not built, stated rather than implied: -// - `:countTokens`. It is a second entry against this protocol rather than another wire of -// this one: no stream, no billable turn, and a `{ totalTokens }` envelope of its own. // - the affinity egress' thought-signature rewrite is here, but the Gemini interceptors are // only partly stages: the four `interceptors/` entries are, and nothing else is. // - the Google-RPC envelope `respond.ts` wrote into a client's own stream for a turn that diff --git a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts index da0fb0ee88..540bdd88fc 100644 --- a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts @@ -13,10 +13,12 @@ // replaced surface wrote neither a usage row nor a performance sample for a measurement, so // there is no `writeSettlement` here and the ending states an empty billed set. // -// There is one wire. Only a native Messages endpoint measures — no translation carries the -// question — so the chain dials directly instead of forking, and the rule that generation -// keeps inside its Messages wire (the system-role rewrite, which speaks about what that -// endpoint accepts) sits in the array here for the same reason it sits there. +// There is one wire, and it is shared. Only an upstream's own Messages endpoint measures — +// no protocol but this one answers "what would this cost" — so the chain dials directly +// instead of forking, and Gemini's `:countTokens` translates into the same wire rather than +// growing one of its own. The rule generation keeps inside its Messages wire (the system-role +// rewrite, which speaks about what that endpoint accepts) is therefore a rule of this wire +// too, and sits with it. // // What it measures is what generation would send. The same three request rules run, in the // order the interceptor chain ran them, and the web-search shim's request half runs above @@ -40,7 +42,7 @@ import { import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; import { renderProtocolError } from '@floway-dev/protocols/common'; import { parseAnthropicBetaHeader, type MessagesPayload } from '@floway-dev/protocols/messages'; import { providerModelOf } from '@floway-dev/provider'; @@ -50,12 +52,12 @@ import { providerModelOf } from '@floway-dev/provider'; * this. */ export const messagesCountTokensTarget = chatTargetPicker(['messages']); -/** What a measurement hands up. It rides at this family's own response key — which is what +/** What a measurement hands up. It rides at a protocol's own response key — which is what * lets the request rules generation runs be the same stages here — and the arm is narrowed - * because nothing in this chain opens a stream. */ -type TokenCount = { readonly kind: 'value'; readonly body: unknown } | Failure; + * because nothing in a counting chain opens a stream. */ +export type TokenCountAnswer = { readonly kind: 'value'; readonly body: unknown } | Failure; -type Counted = { [P in K]: TokenCount }; +type Counted = { [P in K]: TokenCountAnswer }; type M = { [P in K]: MessagesFacts[P] }; @@ -243,6 +245,25 @@ const callMessagesCountTokensUpstream = defineStage< }, }); +/** + * The Messages count-tokens wire, as the chain that dials it. + * + * Every source protocol whose measurement reaches an upstream does it over this endpoint — + * Gemini's `:countTokens` has no wire of its own and arrives here through a translation — so + * a rule that speaks about *this* wire belongs here rather than beside a source's own + * strippers. All four do: the system-role rewrite and the reasoning sentinel state what an + * upstream's Messages endpoint accepts, the billing-attribution scrub states what generation + * would have sent it, and the web-search preparation states the tool shape it would have + * seen. + */ +export const messagesCountTokensWire: readonly Stage[] = [ + prepareMessagesWebSearchRequest, + stripBillingAttributionFromMessages, + disableReasoningOnForcedToolChoiceForMessages, + applyRoleCompatibilityToMessages, + callMessagesCountTokensUpstream, +]; + /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ @@ -271,9 +292,5 @@ export const messagesCountTokensPipeline = (payload: MessagesPayload): Pipeline< owns: [], }), materializeAttempt('request.chat.messages'), - prepareMessagesWebSearchRequest, - stripBillingAttributionFromMessages, - disableReasoningOnForcedToolChoiceForMessages, - applyRoleCompatibilityToMessages, - callMessagesCountTokensUpstream, + ...messagesCountTokensWire, ]); From cd7fec0047ca9b79bc3214788a131817e59f863a Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 09:40:03 +0800 Subject: [PATCH 045/156] feat(responses): bring the WebSocket transport onto the chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last `/v1/responses` entry still on the old surface. A `response.create` frame now opens a run over `responsesServePipeline` exactly as the HTTP route does — entering the pipeline system takes no capability, and a transport outside it is the normal position of a caller. What the transport keeps is what only it can do: its own framing, its own lifecycle, and the connection-local cache the spec makes it evict from. The chain could not serve a WS turn as assembled, in one respect: `emitResponses` rendered a streamed answer as SSE, which is a wire format for an HTTP body rather than something a socket writes. It now takes the framing the entry assembled it with — the same events either way, SSE written from them or handed over for a transport that frames each one itself. Reading them back as parsed SSE was rejected: it would re-parse frames this gateway had just serialized, and a mid-stream failure would reach the client as a stream event instead of the error envelope with a `status` the spec requires. Settlement, telemetry and the failure envelopes are the chain's now. What survives here is the eviction rule, which speaks about this connection rather than about the run, and therefore still reads the frames as they go out. Fixes what that exposed: `meterResponses` learned it had seen the terminal event only from being resumed past it, and the stage that stores the turn's items stops reading there, so the resumption never came — every completed streaming turn settled as a failure and never stamped its model or token counts on the dump. It reads the frame instead. `/v1/responses` over HTTP had the same defect. A turn is recorded as its whole run rather than as a pair of edges, and names itself `WS /v1/responses` so one socket's many turns are told apart. It opens that recording with the session-scoped scheduler, because the runtime's own is only legal while the fetch that returned the 101 is still running. --- .../chat/responses/websocket_test.ts | 187 ++++++----- .../src/data-plane/chat/responses/pipeline.ts | 50 ++- .../data-plane/chat/responses/websocket.ts | 317 +++++++++++------- 3 files changed, 338 insertions(+), 216 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts index 77a131dc45..e23cdb57e2 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts @@ -3,11 +3,11 @@ import { onTestFinished, test, vi } from 'vitest'; import { app } from '../../../../src/app.ts'; import { hashResponsesItem } from '../../../../src/data-plane/chat/responses/items/identity.ts'; -import { responsesServe } from '../../../../src/data-plane/chat/responses/serve.ts'; import { KEEP_ALIVE_EVENT_TYPE } from '../../../../src/data-plane/chat/responses/websocket.ts'; import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../../../src/data-plane/shared/sse.ts'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; -import { edgeRecordOf, installDumpStubs } from '../../../dump/test-fixtures.ts'; +import { tokenCountsFromUsage } from '../../../../src/repo/usage-metrics.ts'; +import { installDumpStubs, runRecordOf } from '../../../dump/test-fixtures.ts'; import { FakeTime } from '../../../test-time.ts'; import { buildCodexUpstreamRecord, codexModels, copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseResponsesResponse } from '../../../test-utils/app.ts'; import { installWorkerWebSocketRuntime, type TestWorkerWebSocket } from '../../../test-utils/worker-websocket.ts'; @@ -226,17 +226,26 @@ test('Responses WebSocket starts capturing on the next turn when dump retention const stored = dumps.stored[0]; assertExists(stored); assertEquals(stored.keyId, apiKey.id); - const request = edgeRecordOf(stored.record).request; - assertEquals(request.method, 'WS'); - assertEquals(request.path, '/v1/responses'); - assertEquals(JSON.parse(new TextDecoder().decode(request.body)), { + // The shape follows the endpoint, and this turn was served through the pipeline: it is + // recorded as its whole run rather than as a pair of edges. One socket carries many + // turns, so each names itself — `WS /v1/responses`, not the `GET` upgrade that opened + // the connection — and the frame that opened it is that run's request, both as the + // bytes the record measured and as the payload the chain ran on. + const record = runRecordOf(stored.record); + assertEquals(record.meta.method, 'WS'); + assertEquals(record.meta.path, '/v1/responses'); + assertEquals(record.meta.requestBytes, new TextEncoder().encode(JSON.stringify({ type: 'response.create', event_id: 'capture-after-enable', response: { model: 'gpt-direct-responses', input: 'capture-after-enable', }, - }); + })).byteLength); + assert( + new TextDecoder().decode(record.events).includes('"capture-after-enable"'), + 'expected the run to record the payload the frame opened the turn with', + ); client.close(); }), ); @@ -290,6 +299,44 @@ test('Responses WebSocket dump responseBytes equals the UTF-8 payload bytes sent ); }); +// A turn on this transport is a run, and a run settles once: the chain reads what the +// upstream metered off its own events as they pass, and the transport settles that reading +// after the last frame has gone out. The rows are filtered to this turn's own key because +// every `setupAppTest` mints a fresh one, and background writes from earlier files in the +// suite resolve against whichever repo was installed last. +test('Responses WebSocket settles a streamed turn once, with what the chain read off it', async () => { + const { apiKey, repo } = await setupAppTest(); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + + await withSuccessfulResponsesUpstream( + async () => await withWorkerWebSocketRuntime(async () => { + const client = await connectResponsesWebSocket(apiKey.key); + await completeResponsesTurn(client, 'settled-turn'); + client.close(); + }), + ); + await flushAsyncWork(); + + const usage = (await repo.usage.listAll()).filter(row => row.keyId === apiKey.id); + assertEquals(usage.length, 1); + assertEquals(usage[0]?.model, 'gpt-direct-responses'); + assertEquals(tokenCountsFromUsage(usage[0]!), { input: 3, output: 5 }); + + const performance = (await repo.performance.listAll()).filter(row => row.keyId === apiKey.id); + assertEquals(performance.length, 1); + assertEquals(performance[0]?.requests, 1); + assertEquals(performance[0]?.errorsNoOutput, 0); + + // The same reading, on the record: a turn that was settled is a turn the dump names the + // upstream's model and token counts on. + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + const meta = runRecordOf(dumps.stored[0]?.record).meta; + assertEquals(meta.model, 'gpt-direct-responses'); + assertEquals(meta.inputTokens, 3); + assertEquals(meta.outputTokens, 5); +}); + test('Responses WebSocket rejects the next turn after its API key is rotated', async () => { const { apiKey, repo } = await setupAppTest(); @@ -1367,78 +1414,68 @@ test('Responses WebSocket aborts the in-flight Responses request when the client ); }); -// The four chat HTTP transports render a mid-attempt throw (interceptor -// bug, translation error, provider-layer JS exception not represented as a ChatServeFailure) through an -// `internalErrorResult(..., ctx.attempt.telemetry)` envelope, -// which internally reaches `recordFailedRequest` and lands an error row -// attributed to the throwing candidate. The WS transport's outer catch -// must do the same: alongside its sendError / dump.failed / dump.finalize, -// it calls `recordFailedRequest(ctx, ctx.attempt.telemetry)` so -// the failure shows up in performance_summary. +// A throw that escapes the run never reached the settlement stage, so the sample +// that stage would have written is the WS transport's outer catch to write — +// alongside its sendError / dump.failed / dump.finalize it calls +// `recordFailedRequest(ctx, ctx.attempt.telemetry)`, and the row lands attributed +// to the candidate that was being dialled when it threw. The throw here is a real +// one: a refusal whose body cannot be read. `callResponsesUpstream` stamps this +// candidate's telemetry before it dials and then reads the refusal's own words, so +// the read's rejection is a mid-attempt throw with attribution already in place. test('Responses WebSocket outer catch records a failed perf sample attributed to the throwing candidate', async () => { - const { apiKey, repo } = await setupAppTest(); + const { apiKey, repo, copilotUpstream } = await setupAppTest(); - // Mirror what responsesServe.generate would have stamped before failing - // — telemetry set for the throwing candidate — then throw. - const generateSpy = vi.spyOn(responsesServe, 'generate').mockImplementation(async ({ ctx }) => { - ctx.attempt.telemetry = { - keyId: apiKey.id, - model: 'gpt-direct-responses', - upstream: 'up_throwing', - operation: 'chat', - runtimeLocation: 'TEST', - }; - throw new Error('simulated mid-attempt provider throw'); - }); - - try { - await withMockedFetch( - async request => { - const url = new URL(request.url); - if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); - if (url.pathname === '/copilot_internal/v2/token') { - return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); - } - if (url.pathname === '/models') { - return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); - const received = waitForMessages(client, messages => messages.length === 1); - client.send(JSON.stringify({ - type: 'response.create', - event_id: 'evt_throw', - response: { model: 'gpt-direct-responses', input: 'hello' }, - })); + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') { + return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); + } + if (url.pathname === '/responses') { + return new Response( + new ReadableStream({ + start(controller) { controller.error(new Error('simulated mid-attempt provider throw')); }, + }), + { status: 400, headers: { 'content-type': 'application/json' } }, + ); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => await withWorkerWebSocketRuntime(async () => { + const client = await connectResponsesWebSocket(apiKey.key); + const received = waitForMessages(client, messages => messages.length === 1); + client.send(JSON.stringify({ + type: 'response.create', + event_id: 'evt_throw', + response: { model: 'gpt-direct-responses', input: 'hello' }, + })); - const [errorMessage] = await received; - assertExists(errorMessage); - assertEquals(errorMessage.type, 'error'); - assertEquals(errorMessage.status, 500); - assertEquals(errorMessage.event_id, 'evt_throw'); - }), - ); + const [errorMessage] = await received; + assertExists(errorMessage); + assertEquals(errorMessage.type, 'error'); + assertEquals(errorMessage.status, 500); + assertEquals(errorMessage.event_id, 'evt_throw'); + }), + ); - await flushAsyncWork(); - - // Filter to the throwing upstream: earlier WS tests in the same file - // schedule background recordFailedRequest calls through the session - // scheduler, and the shared `getRepo()` global resolves them against - // whichever repo `setupAppTest` last installed — so cross-test rows can - // land here. Only the row from the mocked generate is load-bearing for - // this fix. - const perfRows = (await repo.performance.listAll()).filter(row => row.upstream === 'up_throwing'); - assertEquals(perfRows.length, 1); - assertEquals(perfRows[0]?.upstream, 'up_throwing'); - assertEquals(perfRows[0]?.model, 'gpt-direct-responses'); - assertEquals(perfRows[0]?.operation, 'chat'); - assertEquals(perfRows[0]?.errorsNoOutput, 1); - assertEquals(perfRows[0]?.requests, 1); - } finally { - generateSpy.mockRestore(); - } + await flushAsyncWork(); + + // Filter to this turn's own key: earlier WS tests in the same file schedule + // background performance writes through the session scheduler, and the shared + // `getRepo()` global resolves them against whichever repo `setupAppTest` last + // installed — so cross-test rows for the same upstream can land here. Every + // `setupAppTest` mints a fresh api key, which is what tells them apart. + const perfRows = (await repo.performance.listAll()) + .filter(row => row.keyId === apiKey.id && row.upstream === copilotUpstream.id); + assertEquals(perfRows.length, 1); + assertEquals(perfRows[0]?.model, 'gpt-direct-responses'); + assertEquals(perfRows[0]?.operation, 'chat'); + assertEquals(perfRows[0]?.errorsNoOutput, 1); + assertEquals(perfRows[0]?.requests, 1); }); test('Responses WebSocket dispatches each Codex turn with the metadata blob that turn carried', async () => { diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index 61745e5121..ae59c07c9d 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -16,12 +16,14 @@ // do, which is why their interceptor forms stood down on `ctx.targetApi !== 'responses'`, and // position says it here instead. // -// One transport, and everything else this protocol owns is a step of its own, stated here -// rather than implied by its absence: +// Two transports enter here — `POST /v1/responses` and the WebSocket one — and the only thing +// they disagree about is how a streamed answer is framed, which is what `framing` says. Below +// the edge there is nothing to tell them apart: both open a run, both are outside the pipeline +// system, and neither needs a capability to start one. +// +// What this protocol owns and this chain does not is stated here rather than implied by its +// absence: // -// - the WebSocket transport. A second entry against this protocol — its own framing, its -// own lifecycle — and `websocket.ts` still owns it. It reaches the same membrane through -// `serve-prep.ts` and `client-output.ts`, which is why both outlive this chain. // - `/v1/responses/compact`. A second operation over this protocol rather than another // wire under this pipeline, so it stays on `responsesServe.compact`. The dial here asks // for `generate`; what the ending answers with is the branch the provider says it ran, @@ -88,6 +90,12 @@ import { translateResponsesViaChatCompletions, translateResponsesViaMessages } f * translated Chat Completions path. */ export const responsesTarget = chatTargetPicker(['responses', 'messages', 'chat-completions']); +/** How the transport that opened the run frames a streamed answer. Both carry this + * protocol's own events and neither adds or drops one: SSE is the format an HTTP body is + * written in, terminator and all, and a WebSocket turn writes each event as a text frame of + * its own — so the transport that owns the socket takes the events and frames them itself. */ +export type ResponsesStreamFraming = 'sse' | 'events'; + /** What this family adds to the chat space. */ export interface ResponsesFacts extends ChatFacts { /** Server-only state the gateway once attached to an item it emitted, keyed by that @@ -98,9 +106,10 @@ export interface ResponsesFacts extends ChatFacts { * properties, so it would be written into the dump as an empty object and the record would * say the turn hydrated nothing. */ 'request.chat.responses.privatePayloads': readonly (readonly [string, unknown])[]; - /** What the client is actually sent — an object when it asked for one, SSE frames when it - * asked to stream. The edge provides it, so a dump shows what the client received. */ - 'response.chat.responses.rendered': Record | AsyncIterable; + /** What the client is actually sent — an object when it asked for one, and the stream it + * asked to be streamed, in the framing its transport writes. The edge provides it, so a + * dump shows what the client received. */ + 'response.chat.responses.rendered': Record | AsyncIterable | AsyncIterable>; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ 'response.chat.responses.streamedUsage': Promise | null; @@ -121,8 +130,12 @@ type R = { [P in K]: ResponsesFacts[P] }; * The stored-items membrane's other half runs here too, and it takes the client's own * payload rather than the record's, because by the fork the record holds the prepared one — * `previous_response_id` expanded away — and the resource echoes what the client asked with. + * + * A streamed answer is handed on in the framing the transport that opened the run writes. + * Only the last step differs: `renderSSE` is a wire format written over an HTTP body, and a + * transport that frames each event itself takes the events it would have been written from. */ -const emitResponses = (client: CanonicalResponsesPayload) => defineStage< +const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStreamFraming) => defineStage< R<'ingress.chat.responses.wantsStream'>, R<'ingress.chat.responses.wantsStream'>, R<'ingress.chat.responses.wantsStream' | 'response.chat.responses' | 'response.http.headers'>, @@ -206,7 +219,7 @@ const emitResponses = (client: CanonicalResponsesPayload) => defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(renderSSE(frames)), + 'response.chat.responses.rendered': move(framing === 'sse' ? renderSSE(frames) : frames), 'response.http.status': 200, }; }, @@ -527,11 +540,17 @@ const meterResponses = ( } const usage = billableUsageFromResponsesEvent(frame.event); if (usage !== null) reported = usage; + // Read off the frame rather than off having been resumed past it. The stage that + // stores the turn's items stops reading at the terminal event, so the resumption + // never comes: what this loop would learn from it is already true when the frame + // arrives, and the turn ended the way the upstream said it did whether or not + // anything downstream asked for another. + if (isResponsesTerminalEvent(frame.event)) sawTerminal = true; yield frame; // The turn is over, so there is nothing further to read. An upstream that holds the // connection open past its terminal event would otherwise hold the client's stream // open with it; returning here closes the read, which cancels the upstream. - if (isResponsesTerminalEvent(frame.event)) { sawTerminal = true; return; } + if (sawTerminal) return; } // Frames ran out with no terminal event, which is a turn nobody can answer from: the // response was never stated complete, incomplete or failed. @@ -740,14 +759,19 @@ export type ResponsesServeExit = R< | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' >; -export const responsesServePipeline = (payload: CanonicalResponsesPayload): Pipeline => { +export const responsesServePipeline = ( + payload: CanonicalResponsesPayload, + // SSE is what a run is written in when nothing else claims its frames, which is every + // entry over an HTTP body; the WebSocket transport says so because it writes its own. + framing: ResponsesStreamFraming = 'sse', +): Pipeline => { // One cell per run, written by the stage directly above the one that reads it. The // resolver takes its narrowing at assembly, so this is where the prepared payload crosses // from the membrane to the affinity walk; until the membrane has run, what the client sent // is the whole of what is known about the turn. let prepared = payload; return compose('responsesServe', [ - emitResponses(payload), + emitResponses(payload, framing), writeSettlement( handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), handedUp => (handedUp as { 'response.chat.responses.streamedUsage'?: unknown })['response.chat.responses.streamedUsage'] !== null, diff --git a/packages/gateway/src/data-plane/chat/responses/websocket.ts b/packages/gateway/src/data-plane/chat/responses/websocket.ts index cfbc5fa47b..dd587e1f22 100644 --- a/packages/gateway/src/data-plane/chat/responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/responses/websocket.ts @@ -1,25 +1,37 @@ +// The Responses WebSocket transport: a second entry against the same chain `POST +// /v1/responses` runs. One `response.create` frame is one turn and one turn is one run, so +// nothing about a turn outlives the frame that opened it except what the connection itself +// holds — the session's item store and the ids a continuation may name. +// +// Entering the pipeline system takes no capability, so this handler opens a run exactly as a +// Hono route handler does. What it does not do is answer through `serveThrough`: that builds +// an HTTP `Response`, and a turn here writes its own frames to a socket, counts them, and +// closes its own recording. + import type { Context } from 'hono'; -import { wrapResponsesClientEgress } from './client-output.ts'; -import { createResponsesWsSession } from './items/store.ts'; -import { PreviousResponseNotFoundError } from './serve-prep.ts'; -import { responsesServe } from './serve.ts'; +import { createResponsesWsSession, type StatefulResponsesStore } from './items/store.ts'; +import { responsesServePipeline, type ResponsesFacts, type ResponsesServeExit } from './pipeline.ts'; +import { openRunDump } from '../../../dump/run-sink.ts'; import type { TurnDump } from '../../../dump/turn-dump.ts'; import { apiKeyFromContext, authenticateApiKey, type AuthedContext } from '../../../middleware/auth.ts'; +import type { ApiKey } from '../../../repo/types.ts'; import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { takeRequestBody } from '../../shared/request-body.ts'; +import { consoleLogSink } from '../../../runtime/log.ts'; +import { prologueFor, type Ingress } from '../../pipeline/serve.ts'; +import { settleBillable } from '../../pipeline/settlement.ts'; +import { takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS, type StreamCompletion } from '../../shared/sse.ts'; import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; +import type { ChatPrologue } from '../prologue.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { SourceStreamState, eventResultMetadata } from '../shared/respond.ts'; +import { SourceStreamState } from '../shared/respond.ts'; +import { move, run } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import { isJsonMediaType, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { RESPONSES_MISSING_TERMINAL_MESSAGE } from '@floway-dev/protocols/responses'; -import { isResponsesTerminalEvent, type CanonicalResponsesPayload, type ClientResponsesStreamEvent, type ResponsesRequestPayload, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import type { ExecuteResult } from '@floway-dev/provider'; +import { isResponsesTerminalEvent, type CanonicalResponsesPayload, type ClientResponsesStreamEvent, type ResponsesRequestPayload } from '@floway-dev/protocols/responses'; +import type { ModelCandidate } from '@floway-dev/provider'; import { toInternalDebugError } from '@floway-dev/provider'; import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; @@ -122,11 +134,12 @@ const createResponsesWebSocketEvents = (c: AuthedContext): ResponsesWebSocketHan // during the fetch invocation; once the fetch handler returns the 101 // upgrade, subsequent waitUntil calls made from message-event handlers // are silently dropped (the promise never runs, the isolate has no - // registered reason to defer eviction for it). Every per-message background - // task — dump.finalize, settle, recordFailedRequest — would therefore - // lose its write. + // registered reason to defer eviction for it). Every per-turn background + // task — the dump write, the usage row, the performance sample, the drain — + // would therefore lose its write, so this is the scheduler every turn's run + // is opened with rather than the context's. // - // Fix: give the ctx a scheduler that doesn't depend on the fetch's + // Fix: give the run a scheduler that doesn't depend on the fetch's // execution context at all. `sessionScheduler` tracks the task in // `pendingWork`; the isolate stays alive throughout because we register // ONE lifetime promise up-front (while the fetch handler is still @@ -135,11 +148,11 @@ const createResponsesWebSocketEvents = (c: AuthedContext): ResponsesWebSocketHan // // The drain uses a `while (size > 0)` loop rather than a single // `Promise.allSettled(pendingWork)` snapshot: the in-flight message - // handler running at close time may still enqueue a final - // dump.finalize / settle / recordFailedRequest from its finally/catch after - // `sessionClosed` resolves. The loop keeps going until the Set is - // genuinely empty, which is bounded because `closed = true` short- - // circuits future message handlers at the top of `handleClientMessage`. + // handler running at close time may still enqueue a final write from its + // finally/catch after `sessionClosed` resolves. The loop keeps going until + // the Set is genuinely empty, which is bounded because `closed = true` + // short-circuits future message handlers at the top of + // `handleClientMessage`. const pendingWork = new Set>(); let sessionClosedResolve: (() => void) | undefined; const sessionClosed = new Promise(resolve => { sessionClosedResolve = resolve; }); @@ -192,6 +205,66 @@ interface ResponsesWsTurnFailure { fail(status: number, error: Record): void; } +/** + * A turn's own run. + * + * `openChatPrologue` reads the turn's shape off the Hono context, and a turn on this + * transport shares none of that shape with the request that opened the socket. It is a `WS` + * turn on a connection whose upgrade was a `GET`; its body is the frame that just arrived + * rather than the upgrade's, which had none; it owns the abort controller the session cancels + * it with; and its background work outlives the fetch that returned the 101, so the scheduler + * it settles and records through is the session's rather than that fetch's. What it does not + * differ in is the services a chat run is given, which is why it hands those back in the shape + * every chat entry hands them over in. + */ +const openResponsesWebSocketTurn = ( + c: AuthedContext, + turn: { + readonly payload: CanonicalResponsesPayload; + readonly body: RequestBody; + readonly headers: Ingress['headers']; + readonly downstreamAbortController: AbortController; + readonly backgroundScheduler: BackgroundScheduler; + readonly store: (apiKey: ApiKey, requestStartedAt: number) => StatefulResponsesStore; + }, +): ChatPrologue => { + // The frame is this turn's request body, so an operator reading the dashboard sees the + // exact `response.create` that opened it, under its own `WS /v1/responses` row rather than + // under the upgrade that carried it. + const dump = openRunDump( + apiKeyFromContext(c), + { method: 'WS', path: new URL(c.req.raw.url).pathname, body: turn.body }, + turn.backgroundScheduler, + ); + const gateway = createChatGatewayCtxFromHono(c, { + wantsStream: true, + model: turn.payload.model, + requestBody: takeRequestBody(turn.body), + downstreamAbortController: turn.downstreamAbortController, + backgroundScheduler: turn.backgroundScheduler, + dump, + }, turn.store); + const base = prologueFor(gateway, { body: turn.body, headers: turn.headers }, dump); + + let materialize: ((candidate: ModelCandidate) => unknown) | undefined; + return { + ...base, + gateway, + services: { + ...base.services, + gateway, + rememberChatSelection: payloadFor => { materialize = payloadFor; }, + chatPayloadFor: selector => { + if (materialize === undefined) { + throw new Error('chatPayloadFor: nothing was resolved in this run; the selector did not come from it'); + } + return materialize(base.services.resolveAttempt(selector)); + }, + selectAffinity: candidate => { gateway.affinity.select(candidate); }, + }, + }; +}; + const handleClientMessage = async ( c: AuthedContext, socket: ResponsesWebSocketSocket, @@ -207,20 +280,17 @@ const handleClientMessage = async ( let ctx: ChatGatewayCtx | undefined; let previousResponseId: string | undefined; - // "If a continuation turn fails with a `4xx` or `5xx` error, the server MUST - // evict the referenced `previous_response_id` from the connection-local - // cache. A later attempt to continue from that evicted `store=false` - // response ID on the same connection MUST fail with - // `previous_response_not_found`." + // "If a continuation turn fails with a `4xx` or `5xx` error, the server MUST evict the + // referenced `previous_response_id` from the connection-local cache. A later attempt to + // continue from that evicted `store=false` response ID on the same connection MUST fail + // with `previous_response_not_found`." // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L127 // - // Eviction is wired at two points because a failing turn can leave through - // two exits that do not share a path. `fail` covers the turn that answers - // the client with an error envelope, including the `api-error` and - // `internal-error` results, which return before the streaming loop is - // entered; the loop's `finally` covers the turn that ends failed without - // one. A throw inside the loop is the single path that takes both, and - // `Map.delete` makes the second call inert. + // Eviction is wired at two points because a failing turn can leave through two exits that + // do not share a path. `fail` covers the turn that answers the client with an error + // envelope, including the refusals the chain hands back as a body rather than a stream; the + // streaming loop's `finally` covers the turn that ends failed without one. A throw inside + // the loop is the single path that takes both, and `Map.delete` makes the second call inert. const turnFailure: ResponsesWsTurnFailure = { evict: () => { if (ctx === undefined || previousResponseId === undefined) return; @@ -233,11 +303,10 @@ const handleClientMessage = async ( }; try { - // Capture raw frame bytes up front so they're available as the dump's - // request body when `ctx` is constructed below. Payloads that fail to - // parse never reach ctx construction, so no dump record is emitted for - // them — there is no api-key-scoped turn to attribute them to. - const requestBody = { bytes: wsDataToBytes(data), streamError: null }; + // Capture raw frame bytes up front so they're available as the run's request body when + // the turn is opened below. Payloads that fail to parse never reach that point, so no + // dump record is emitted for them — there is no api-key-scoped turn to attribute them to. + const requestBody: RequestBody = { bytes: wsDataToBytes(data), streamError: null }; if (!(await authenticateApiKey(c, authenticatedRawKey))) { turnFailure.fail(401, { type: 'authentication_error', @@ -270,41 +339,36 @@ const handleClientMessage = async ( : Object.fromEntries(Object.entries(message).filter(([key]) => key !== 'type' && key !== 'event_id')); const payload = responsesPayloadFromClientSource(source); previousResponseId = payload.previous_response_id ?? undefined; - ctx = createChatGatewayCtxFromHono(c, { - wantsStream: true, + + const prologue = openResponsesWebSocketTurn(c, { + payload, + body: requestBody, + // The upgrade's own headers are the connection's, and every turn on it is dialled with + // them: this transport has no per-turn header surface, so what a client wants a later + // turn to carry it carries in the frame body instead. + headers: [...c.req.raw.headers], downstreamAbortController, - // The WS upgrade has no HTTP body; the dump's request body is the - // per-turn JSON frame bytes so an operator reading the dashboard - // sees the exact `response.create` payload the client sent. - requestBody: takeRequestBody(requestBody), - method: 'WS', - model: payload.model, backgroundScheduler, - }, (apiKey, requestStartedAt) => session.createStore(apiKey, requestStartedAt, payload.store ?? undefined)); - - let result; - try { - result = await responsesServe.generate({ payload, ctx, headers: inboundHeaders(c) }); - } catch (error) { - if (signal.aborted || isClosed()) return; - // The HTTP entry renders this verbatim envelope as a 400; WS surfaces the - // same body nested under the spec's WebSocket error envelope so clients - // can still compare error.message byte-for-byte against upstream. - if (error instanceof PreviousResponseNotFoundError) { - turnFailure.fail(400, { - message: error.message, - type: 'invalid_request_error', - param: 'previous_response_id', - code: 'previous_response_not_found', - }); - ctx.dump?.failed(error); - ctx.dump?.finalize(400, []); - return; - } - throw error; - } - - await respondResponsesWebSocket({ socket, eventId, signal, isClosed, result, ctx, payload, turnFailure }); + store: (apiKey, requestStartedAt) => session.createStore(apiKey, requestStartedAt, payload.store ?? undefined), + }); + ctx = prologue.gateway; + + const { facts, drain } = await run( + // The transport frames its own answer, so the edge hands up the events rather than the + // SSE a body would have been written from. + responsesServePipeline(payload, 'events'), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'responses', + // A turn on this transport always streams, whatever the client wrote. + 'ingress.chat.responses.wantsStream': true, + 'request.chat.responses': payload, + 'serve.model': payload.model, + }) as never, + prologue.services as never, + ); + + await respondResponsesWebSocket({ socket, eventId, signal, isClosed, prologue, facts, drain, turnFailure }); } catch (error) { if (signal.aborted || isClosed()) return; if (error instanceof TranslatorInputError) { @@ -326,14 +390,12 @@ const handleClientMessage = async ( } turnFailure.fail(500, serverErrorEnvelope(error)); if (ctx !== undefined) { - // Mid-attempt throws (interceptor bug, translation error, provider-layer JS - // exception not represented as a ChatServeFailure) never reach the - // respondResponsesWebSocket result branches, so their `recordFailedRequest` - // call would be skipped. Attribute the failure to the last upstream stamped - // synchronously by `responsesServe.generate`, matching the HTTP transports. + // A throw that escapes the run never reached the settlement stage, so the sample that + // stage would have written is written here instead — attributed to the last upstream + // the chain stamped, which is the one that was being dialled when it threw. recordFailedRequest(ctx, ctx.attempt.telemetry); ctx.dump?.failed(error); - ctx.dump?.finalize(500, []); + ctx.dump?.finalize(500, 0); } } }; @@ -358,30 +420,38 @@ const validateClientMessage = (parsed: unknown): ResponsesWebSocketClientEvent = const responsesPayloadFromClientSource = (source: object): CanonicalResponsesPayload => ({ ...canonicalizeResponsesPayload(source as ResponsesRequestPayload), stream: true }); +/** The chain hands its answer up at one key, and this entry assembled it to hand up events + * rather than the SSE frames an HTTP body is written from — so what is not a stream at that + * key is the one object arm this transport can see. */ +const isRenderedStream = ( + rendered: ResponsesFacts['response.chat.responses.rendered'], +): rendered is AsyncIterable> => Symbol.asyncIterator in rendered; + const respondResponsesWebSocket = async (input: { readonly socket: ResponsesWebSocketSocket; readonly eventId: string | undefined; readonly signal: AbortSignal; readonly isClosed: () => boolean; - readonly result: ExecuteResult>; - readonly ctx: ChatGatewayCtx; - readonly payload: CanonicalResponsesPayload; + readonly prologue: ChatPrologue; + readonly facts: ResponsesServeExit; + readonly drain: () => Promise; readonly turnFailure: ResponsesWsTurnFailure; }): Promise => { - const { socket, eventId, signal, isClosed, result, ctx, payload, turnFailure } = input; - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - turnFailure.fail(result.status, normalizeErrorBody(parseMaybeJson(result.body, result.headers), result.status)); - ctx.dump?.finalize(result.status, []); - return; - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - turnFailure.fail(result.status, internalErrorEnvelope(result.error)); - ctx.dump?.finalize(result.status, []); + const { socket, eventId, signal, isClosed, prologue, facts, drain, turnFailure } = input; + const ctx = prologue.gateway; + const status = facts['response.http.status']; + const rendered = facts['response.chat.responses.rendered']; + + if (!isRenderedStream(rendered)) { + // A body at the rendered key is a refusal — the upstream's own, or one the chain made + // before reaching an upstream. Nothing else can arrive there on this transport: a turn + // here always asks to stream, and the one non-refusal body the chain can produce is a + // compaction envelope, which no `response.create` frame has a way to ask for. It goes out + // under the spec's WebSocket error envelope so a client can still compare error.message + // byte-for-byte against upstream. Nothing is left to read, so releasing starts at once. + prologue.services.background(drain()); + turnFailure.fail(status, normalizeErrorBody(rendered, status)); + ctx.dump?.finalize(status, 0); return; } @@ -389,9 +459,7 @@ const respondResponsesWebSocket = async (input: { let completion: StreamCompletion = 'error'; try { let terminalEvent: ClientResponsesStreamEvent | undefined; - const observed = observeResponsesWebSocketFrames(result.events, state, ctx); - const output = wrapResponsesClientEgress(observed, ctx, payload); - const iterator = output[Symbol.asyncIterator](); + const iterator = observeResponsesWebSocketFrames(rendered, state, ctx)[Symbol.asyncIterator](); let pendingNext = pendingWsFrameResult(iterator.next()); let completed = false; let stoppedByDownstream = false; @@ -550,28 +618,37 @@ const respondResponsesWebSocket = async (input: { state.failed = true; turnFailure.fail(500, serverErrorEnvelope(error)); } finally { - const metadata = await eventResultMetadata(result); + // Writing the frames to the socket *is* releasing the body they came from, so the drain + // waits for that to finish. A client that stopped reading still gets here, which is what + // leaves nothing open behind it. + await drain(); + // What the turn billed, as the chain read it off the upstream's own events while they + // passed. The answer is already on the socket by now, so it is settled here rather than + // handed to the background the way an entry that returns a response has to. + const streamedUsage = facts['response.chat.responses.streamedUsage']; + if (streamedUsage !== null) { + const outcome = await streamedUsage; + settleBillable({ ...prologue.services, log: consoleLogSink }, outcome.billable, outcome.failed); + } const failed = state.failedAfter(completion); if (failed) { - // `fail` cannot carry the eviction for every failed turn: one that - // streamed an `error` or `response.failed` terminal answered the client - // with an event rather than an error envelope, and one the client - // abandoned before the terminal event answered with nothing at all. - // Both settle here. For the abandoned turn the eviction is inert — the + // `fail` cannot carry the eviction for every failed turn: one that streamed an `error` + // or `response.failed` terminal answered the client with an event rather than an error + // envelope, and one the client abandoned before the terminal event answered with + // nothing at all. Both settle here. For the abandoned turn the eviction is inert — the // connection-local cache dies with the socket. turnFailure.evict(); ctx.dump?.failed(`responses ws turn failed (completion=${completion}, source-failed=${state.failed})`); - } else ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - ctx.dump?.finalize(failed ? 500 : 200, []); - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); + } + ctx.dump?.finalize(failed ? 500 : 200, 0); } }; const observeResponsesWebSocketFrames = async function* ( - frames: AsyncIterable>, + frames: AsyncIterable>, state: SourceStreamState, ctx: ChatGatewayCtx, -): AsyncGenerator> { +): AsyncGenerator> { for await (const frame of frames) { ctx.dump?.frame(frame); if (frame.type === 'event') { @@ -650,31 +727,15 @@ const createDownstreamSequence = (): DownstreamSequence => { }; }; -const parseMaybeJson = (body: Uint8Array, headers: Headers): unknown => { - const text = new TextDecoder().decode(body); - if (!isJsonMediaType(headers.get('content-type'))) return { message: text }; - try { - return JSON.parse(text) as unknown; - } catch { - return { message: text }; - } -}; - -const internalErrorEnvelope = (error: Extract>, { type: 'internal-error' }>['error']): Record => ({ - type: error.type, - code: error.type, - name: error.name, - message: error.message, - stack: error.stack, - cause: error.cause, - target_api: error.target_api, -}); - const serverErrorEnvelope = (error: unknown): Record => ({ ...toInternalDebugError(error), code: 'internal_error', }); +/** The rendered refusal, as this transport's envelope carries one. The chain already answered + * in the words a refusal was made in — the upstream's own body when it sent one, and the + * gateway's own envelope when the refusal was its — so what is left is to state the two + * fields the WebSocket envelope requires of every error whatever wrote it. */ const normalizeErrorBody = (body: unknown, status: number): Record => { const source = body && typeof body === 'object' && 'error' in body && typeof (body as { error?: unknown }).error === 'object' ? (body as { error: Record }).error From afbf1966bf5d50992db3fca8d3d29089e8d4dd1b Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 09:45:35 +0800 Subject: [PATCH 046/156] fix(chat): hand a chat run's events somewhere to be written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opening a chat run built the context from one options object and then opened the prologue without the recording those options had started, so every pipelined chat turn over HTTP wrote a record holding its metadata and no events — a key with retention configured paid for a recording of nothing. The WebSocket entry passed its own sink and was unaffected, which is why the gap survived: the two entries share this function and only one of them used what it returns. --- .../chat/responses/run-dump_test.ts | 69 +++++++++++++++++++ .../gateway/src/data-plane/chat/prologue.ts | 10 ++- 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/responses/run-dump_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/responses/run-dump_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/run-dump_test.ts new file mode 100644 index 0000000000..9d9990866c --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/responses/run-dump_test.ts @@ -0,0 +1,69 @@ +// A pipelined chat turn over HTTP is recorded as its whole run, the same way the WebSocket +// transport records one. The two entries share `openChatPrologue`, and the sink it opens is +// the only thing that turns a recording into events — without it a key with retention +// configured pays for a record holding nothing but its metadata. + +import { test, vi } from 'vitest'; + +import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../../dump/test-fixtures.ts'; +import { copilotModels, flushAsyncWork, requestApp, setupAppTest, sseResponsesResponse } from '../../../test-utils/app.ts'; +import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const withResponsesUpstream = async (run: () => Promise): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') { + return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); + } + if (url.pathname === '/responses') { + return sseResponsesResponse({ + id: 'resp_run_dump', + object: 'response', + model: 'gpt-direct-responses', + status: 'completed', + output: [], + output_text: 'done', + usage: { input_tokens: 3, output_tokens: 5, total_tokens: 8 }, + }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + run, + ); + +test('POST /v1/responses records the run it was, not an empty record', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + + await withResponsesUpstream(async () => { + const response = await requestApp('/v1/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-direct-responses', input: 'hello', stream: false }), + }); + assertEquals(response.status, 200); + await response.json(); + }); + + await flushAsyncWork(); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + + const stored = dumps.stored[0]; + assertExists(stored); + const record = runRecordOf(stored.record); + assertEquals(record.meta.path, '/v1/responses'); + + // The events are the point: a recording whose sink was never handed to the runner still + // produces a record, and it holds nothing. Every run enters and leaves at least one stage. + const events = eventsOf(record); + assertEquals(events.length > 0, true); + assertEquals(events.some(event => event.type === 'stage.entered'), true); + assertEquals(events.some(event => event.type === 'stage.leaved'), true); +}); diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts index c00ba96dc2..d0c3f58d3d 100644 --- a/packages/gateway/src/data-plane/chat/prologue.ts +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -12,7 +12,7 @@ import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from './shared/gate import type { ChatServices } from './stages.ts'; import type { ApiKey } from '../../repo/types.ts'; import type { AttemptSelector } from '../pipeline/facts.ts'; -import { gatewayCtxOptions, prologueFor, type Ingress, type Prologue } from '../pipeline/serve.ts'; +import { gatewayCtxOptions, prologueFor, runDumpOf, type Ingress, type Prologue } from '../pipeline/serve.ts'; import type { StatefulResponsesStore } from './responses/items/store.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import type { ModelCandidate } from '@floway-dev/provider'; @@ -35,8 +35,12 @@ export const openChatPrologue = ( ): ChatPrologue => { // One context, built once: the body is read once and `takeRequestBody` empties what it was // given, so the chat context is what this run *has* rather than a second one over it. - const gateway = createChatGatewayCtxFromHono(c, gatewayCtxOptions(c, ingress, options), options.storeFactory); - const base = prologueFor(gateway, ingress); + // One options object, read twice: the context is built from it, and the run recording it + // opened is what the runner's events are written to. Building the options again would take + // the body a second time and hand the dump an empty buffer. + const ctxOptions = gatewayCtxOptions(c, ingress, options); + const gateway = createChatGatewayCtxFromHono(c, ctxOptions, options.storeFactory); + const base = prologueFor(gateway, ingress, runDumpOf(ctxOptions)); let materialize: ((candidate: ModelCandidate) => unknown) | undefined; return { From db05b8be1e6e6e0b5e35d7586660c5b00af0f1f9 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 09:52:46 +0800 Subject: [PATCH 047/156] feat(responses): serve /v1/responses/compact through its own chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compaction is a second operation over this protocol, not another wire under the generate pipeline. Everything above the ending it shares with generation, stage for stage — the stored-items membrane, the narrowing, the four request rules — because a compaction is routed, pinned and rewritten exactly as a turn is. The ending is where they part, and it is where the action pivot went. Two wires. An upstream whose own endpoint compacts is dialled with `compact` and answers one envelope, expanded into the events the stateful half reads so the turn's items are stored under an id this gateway minted. An upstream with no compaction wire — every Messages and Chat Completions candidate, and any Responses candidate an operator opted in with `responses-compact-shim` — gets a stage that rewrites the turn into the compactor's own, hands into the ordinary generate fork below it, and packs the summary into an envelope of this gateway's own on the way back. Nothing needs an action to travel in the record: one wire dials `compact`, the other dials `generate` under a stage that folds the answer. The rewrite itself stays in one place. `summarizationTurnFor` is lifted out of the interceptor and shared, so the simulated compaction is byte-identical whichever entry asked for it, and the interceptor form keeps serving the WebSocket generate path that still reaches it. Two deliberate differences from the replaced entry. A summarization that closed no assistant text is now a candidate the fork moves past rather than a throw that ends the request — the next upstream may well summarize. And the verdict a compaction carries in its own `status` is folded into the accounting at the edge, which is where the completed resource comes into existence and therefore the one reader that can state it. Stated gap: the server-tool shim is still only an interceptor, so a compaction whose caller declared a server tool is summarized without the ReAct loop around it — the same gap the generate chain already states. --- .../data-plane/chat/responses-compact_test.ts | 392 +++++++++++++++ .../src/data-plane/chat/responses/compact.ts | 459 ++++++++++++++++++ .../src/data-plane/chat/responses/http.ts | 170 ++----- .../responses/interceptors/compact-shim.ts | 38 +- .../src/data-plane/chat/responses/pipeline.ts | 58 ++- 5 files changed, 955 insertions(+), 162 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts create mode 100644 packages/gateway/src/data-plane/chat/responses/compact.ts diff --git a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts new file mode 100644 index 0000000000..5eae1dfa69 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts @@ -0,0 +1,392 @@ +// The Responses compaction chain, run. Assembly is checked where every family's is; what is +// written down here is what only running it can say — that an upstream whose own endpoint +// compacts is asked to and answered under an id this gateway minted, that one with no +// compaction wire is sent the compactor's turn over the wires generation would have used and +// answered with an envelope of this gateway's own, that a prior simulated compaction the +// client echoed back is expanded before the history is summarized, that a summarization +// producing nothing is a candidate the fork moves past, and that a compaction whose turn +// failed settles as a failure. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { responsesCompactPipeline } from '../../../src/data-plane/chat/responses/compact.ts'; +import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/responses/interceptors/compact-shim.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { ModelEndpoints } from '@floway-dev/protocols/common'; +import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; +import type { CanonicalResponsesPayload, ResponsesCompactionResult, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +const candidate = ( + calls: { + callResponses?: (model: unknown, body: unknown, action: unknown) => Promise; + callMessages?: (model: unknown, body: unknown) => Promise>; + }, + overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, +): ModelCandidate => { + const upstreamId = overrides.upstreamId ?? 'up_a'; + const endpoints = overrides.endpoints ?? { responses: {} }; + const enabledFlags = overrides.enabledFlags ?? new Set(); + return { + provider: { + upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider(calls as never), + }, + model: stubInternalModel( + { + id: 'responses-model', + endpoints, + limits: { max_output_tokens: 4096 }, + providerModels: { [upstreamId]: stubProviderModel({ id: 'responses-model', endpoints, enabledFlags }) }, + }, + upstreamId, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const USAGE = { input_tokens: 12, output_tokens: 3, total_tokens: 15 }; + +/** What a native compaction upstream answers with: `CompactResource` states no `status`, no + * `model` and no `error`. */ +const compaction = (overrides: Partial = {}): ResponsesCompactionResult => ({ + id: 'resp_upstream_compaction', + object: 'response.compaction', + output: [{ type: 'compaction', id: 'cmp_1', encrypted_content: 'OPAQUE_NATIVE_BLOB' } as unknown as never], + usage: USAGE, + ...overrides, +}); + +const compacts = ( + seen: { body?: Record; action?: unknown }, + overrides: Partial = {}, +) => async (_model: unknown, body: unknown, action: unknown): Promise => { + seen.body = body as Record; + seen.action = action; + return { action: 'compact', ok: true, result: compaction(overrides), modelKey: 'responses-model-key' }; +}; + +/** A generate turn over the Responses wire that produces one assistant message. */ +const generates = (text: string, seen: { body?: Record; action?: unknown } = {}) => + async (_model: unknown, body: unknown, action: unknown): Promise => { + seen.body = body as Record; + seen.action = action; + const response: ResponsesResult = { + id: 'resp_summary', + object: 'response', + model: 'responses-model', + status: 'completed', + output: [{ + type: 'message', id: 'msg_1', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text, annotations: [] }], + }], + error: null, + incomplete_details: null, + usage: USAGE, + }; + return { + action: 'generate', ok: true, modelKey: 'responses-model-key', headers: new Headers(), + events: (async function* () { + yield { type: 'event' as const, event: { type: 'response.completed', sequence_number: 0, response } as ResponsesStreamEvent }; + yield { type: 'done' as const }; + })(), + }; + }; + +/** The same turn over the Messages wire, which is the only way a Messages-only candidate can + * be reached: the compaction crosses into that protocol and its frames come back translated. */ +const messagesTurn = (text: string, seen: { body?: Record } = {}) => + async (_model: unknown, body: unknown): Promise> => { + seen.body = body as Record; + const events: unknown[] = [ + { + type: 'message_start', + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'responses-model', + stop_reason: null, stop_sequence: null, usage: { input_tokens: 12, output_tokens: 0 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 3 } }, + { type: 'message_stop' }, + ]; + return { + ok: true, modelKey: 'responses-model-key', headers: new Headers(), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event: event as MessagesStreamEvent }; + yield { type: 'done' as const }; + })(), + }; + }; + +const payload = { + model: 'responses-model', + input: [{ type: 'message', role: 'user', content: 'a long conversation' }], + store: false, +} as unknown as CanonicalResponsesPayload; + +const settlements: { billable: unknown; failed: boolean }[] = []; + +let gateway = mockChatGatewayCtx({ wantsStream: false }); + +const compact = async (request: CanonicalResponsesPayload = payload) => { + gateway = mockChatGatewayCtx({ wantsStream: false }); + const outcome = await run( + responsesCompactPipeline(request), + move({ + 'ingress.http.headers': [] as readonly (readonly [string, string])[], + 'ingress.chat.sourceProtocol': 'responses', + 'request.chat.responses': request, + 'serve.model': request.model, + }) as never, + { + gateway, + background: () => {}, + rememberCandidates: () => {}, + rememberChatSelection: () => {}, + chatPayloadFor: () => request, + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, + ); + // What the epilogue would settle from, once the run has answered. + const pending = outcome.facts['response.chat.responses.streamedUsage']; + if (pending !== null) settlements.push(await pending); + return outcome; +}; + +const rendered = (facts: Record): Record => + facts['response.chat.responses.rendered'] as Record; + +/** The summary a simulated compaction packed into the item the client is handed. The turn's + * own state is sealed around it on the way out — that is what pins the next turn to the + * upstream that issued this one — so the carrier is opened before the blob is read. */ +const summaryIn = async (encryptedContent: string): Promise => { + const carrier = await gateway.affinity.codec.unwrap(encryptedContent, 'responses.compaction.encrypted_content'); + if (carrier.kind !== 'owned' || carrier.value === undefined) throw new Error('expected the turn-s own carrier around the blob'); + const items = decodeBase64UrlJson(carrier.value) as { content: { text: string }[] }[] | null; + if (items === null) throw new Error('expected a shim-encoded compaction blob'); + return items[0]!.content[0]!.text; +}; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + settlements.length = 0; + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + responsesSnapshots: { lookup: async () => null, put: async () => {} }, + responsesItems: { lookupMany: async () => [], putMany: async () => {} }, + } as never); +}); + +describe('the responses compaction chain', () => { + // An upstream whose own endpoint compacts is asked to compact. What comes back is expanded + // into the events the stateful half reads, which is what replaces the upstream's response id + // with one this gateway minted and completes the resource this endpoint answers with. + it('dials a compaction, and answers the compaction resource under its own id', async () => { + const seen: { body?: Record; action?: unknown } = {}; + resolves([candidate({ callResponses: compacts(seen) })]); + + const { facts } = await compact(); + + expect(seen.action).toBe('compact'); + expect(facts['response.http.status']).toBe(200); + expect(rendered(facts).object).toBe('response.compaction'); + expect(rendered(facts).id).not.toBe('resp_upstream_compaction'); + expect(rendered(facts).created_at).toBeTypeOf('number'); + expect(rendered(facts).usage).toMatchObject({ input_tokens: 12, output_tokens: 3, total_tokens: 15 }); + // A compaction is not a response resource, so none of the twenty-odd keys that one + // requires is decorated onto it. + expect(rendered(facts)).not.toHaveProperty('tools'); + expect(rendered(facts)).not.toHaveProperty('truncation'); + }); + + // Neither field belongs on the compaction endpoint: `store` is a gateway-only snapshot hint + // it rejects, and `stream` describes a delivery this operation does not make. + it('sends neither store nor stream on the compaction wire', async () => { + const seen: { body?: Record } = {}; + resolves([candidate({ callResponses: compacts(seen) })]); + + await compact({ ...payload, stream: false } as unknown as CanonicalResponsesPayload); + + expect(seen.body).not.toHaveProperty('store'); + expect(seen.body).not.toHaveProperty('stream'); + expect(seen.body).not.toHaveProperty('model'); + }); + + // No translation carries a compaction, so a Messages candidate has none to dial. The turn + // it is sent instead is an ordinary generate turn against the compactor's prompt, over the + // wire generation would have used — which is the whole of what the action pivot was. + it('simulates a compaction over a candidate with no compaction wire', async () => { + const seen: { body?: Record } = {}; + resolves([candidate({ callMessages: messagesTurn('CONDENSED SUMMARY', seen) }, { endpoints: { messages: {} } })]); + + const { facts } = await compact({ + ...payload, + input: [ + { type: 'message', role: 'user', content: 'real history' }, + { type: 'compaction_trigger' }, + ], + } as unknown as CanonicalResponsesPayload); + + // The compactor's prompt reached the upstream, and the control item that asked for the + // compaction did not. + const sent = JSON.stringify(seen.body); + expect(sent).toContain('CONTEXT CHECKPOINT COMPACTION'); + expect(sent).not.toContain('compaction_trigger'); + + expect(facts['response.http.status']).toBe(200); + expect(rendered(facts).object).toBe('response.compaction'); + const output = rendered(facts).output as { type: string; encrypted_content: string }[]; + expect(output[0]!.type).toBe('compaction'); + expect(await summaryIn(output[0]!.encrypted_content)).toBe(`${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); + }); + + // The flag is the operator's opt-in for an upstream that would answer a compaction itself. + it('simulates rather than dials when the operator asked it to', async () => { + const seen: { body?: Record; action?: unknown } = {}; + resolves([candidate( + { callResponses: generates('SIMULATED SUMMARY', seen) }, + { enabledFlags: new Set(['responses-compact-shim']) }, + )]); + + const { facts } = await compact(); + + expect(seen.action).toBe('generate'); + // The ephemeral summarization turn is not persisted in the upstream's own history. + expect(seen.body).toMatchObject({ store: false }); + expect(rendered(facts).object).toBe('response.compaction'); + const output = rendered(facts).output as { encrypted_content: string }[]; + expect(await summaryIn(output[0]!.encrypted_content)).toBe(`${SUMMARY_PREFIX}\nSIMULATED SUMMARY`); + }); + + // A compaction this gateway synthesized carries the history it stood for, so a turn that + // echoes it back is summarized from that history rather than from an opaque blob. + it('expands a compaction it wrote before summarizing again', async () => { + const seen: { body?: Record } = {}; + const encoded = encodeBase64UrlJson([ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'THE EARLIER HISTORY' }] }, + ]); + resolves([candidate( + { callResponses: generates('S', seen) }, + { enabledFlags: new Set(['responses-compact-shim']) }, + )]); + + await compact({ + ...payload, + input: [{ type: 'compaction', id: 'cmp_prior', encrypted_content: encoded }], + } as unknown as CanonicalResponsesPayload); + + expect(JSON.stringify(seen.body)).toContain('THE EARLIER HISTORY'); + }); + + // The blob is the whole of what the next turn inherits, so a summarization that closed no + // text produced nothing to inherit — a candidate that did not do the job rather than a + // fault that ends the request. + it('fails a summarization that produced no text over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate( + { callResponses: async (...args) => { tried.push('mute'); return await generates('')(...args as [unknown, unknown, unknown]); } }, + { upstreamId: 'up_mute', enabledFlags: new Set(['responses-compact-shim']) }, + ), + candidate( + { callResponses: async (...args) => { tried.push('talkative'); return await generates('A SUMMARY')(...args as [unknown, unknown, unknown]); } }, + { upstreamId: 'up_talkative', enabledFlags: new Set(['responses-compact-shim']) }, + ), + ]); + + const { facts } = await compact(); + + expect(tried).toEqual(['mute', 'talkative']); + expect(facts['response.http.status']).toBe(200); + expect(rendered(facts).object).toBe('response.compaction'); + }); + + it('fails a refused compaction over to the next candidate, and keeps the last refusal-s own status', async () => { + resolves([ + candidate({ + callResponses: async (): Promise => ({ + action: 'compact', ok: false, modelKey: 'k', + response: new Response(JSON.stringify({ error: { message: 'slow down', type: 'rate_limit' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + }), + }, { upstreamId: 'up_busy' }), + candidate({ + callResponses: async (): Promise => ({ + action: 'compact', ok: false, modelKey: 'k', + response: new Response(JSON.stringify({ error: { message: 'still no', type: 'rate_limit' } }), { + status: 503, headers: { 'content-type': 'application/json' }, + }), + }), + }, { upstreamId: 'up_busier' }), + ]); + + const { facts } = await compact(); + + expect(facts['response.http.status']).toBe(503); + expect(rendered(facts)).toEqual({ error: { message: 'still no', type: 'rate_limit' } }); + }); + + // `Usage` has no `null` alternative on this resource and zeros are not its spelling, so an + // upstream that stated no counts has left the gateway unable to answer. + it('reports a compaction the upstream stated no counts for as the gateway-s own failure', async () => { + resolves([candidate({ callResponses: compacts({}, { usage: undefined }) })]); + + const { facts } = await compact(); + + expect(facts['response.http.status']).toBe(502); + const body = rendered(facts).error as { type: string; message: string }; + expect(body.type).toBe('internal_error'); + expect(body.message).toContain('reported no token usage'); + }); + + // `status` is not a key the compaction resource declares — it rides through from the turn + // the upstream actually ran — and a compaction that surfaced as failed belongs in the error + // column rather than masquerading as a success. + it('settles a compaction whose turn failed as a failure', async () => { + resolves([candidate({ callResponses: compacts({}, { status: 'failed' } as Partial) })]); + + await compact(); + + expect(settlements).toEqual([{ billable: expect.anything(), failed: true }]); + }); + + it('settles a compaction that ran as a success, billing what its envelope stated', async () => { + resolves([candidate({ callResponses: compacts({}) })]); + + await compact(); + + expect(settlements).toHaveLength(1); + expect(settlements[0]!.failed).toBe(false); + expect(settlements[0]!.billable).toMatchObject([ + { quantities: { input_tokens: '12', output_tokens: '3' } }, + ]); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/responses/compact.ts b/packages/gateway/src/data-plane/chat/responses/compact.ts new file mode 100644 index 0000000000..910848f234 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/responses/compact.ts @@ -0,0 +1,459 @@ +// `/v1/responses/compact` as a pipeline. +// +// emitResponsesCompaction the edge: persists what the turn emitted and completes the resource +// writeSettlement above the fork, so a run bills once however many wires it tried +// hydrateStoredItems the stored-items membrane, on the way in +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// materializeAttempt puts the payload this candidate is owed into the record +// beginStoredAttempt reseeds the store's per-attempt scratchpad +// this family's request rules, the same four generation runs above its fork +// dialResponsesCompaction the ending: dials a compaction, or simulates one +// +// A second **operation** over this protocol rather than another wire under +// `responsesServePipeline`, which is why it is a chain of its own. Everything above the +// ending it shares with generation, stage for stage, because a compaction is routed, pinned +// and rewritten exactly as a turn is: the same membrane hydrates it, the same narrowing +// orders its candidates, the same four rules shape the body that goes out. +// +// The ending is where the two part. Two wires, both handing up `response.chat.responses`: +// +// - the compaction wire. An upstream whose own `/responses` endpoint compacts is asked to, +// and answers with one envelope; the synthetic frames it becomes are what the stateful +// half above persists items and a response id from. +// - the simulation. An upstream with no compaction wire of its own — every Messages and +// Chat Completions candidate, and any Responses candidate an operator opted in with +// `responses-compact-shim` — is sent an ordinary generate turn against the compactor's +// prompt over the wires generation would have used, and the summary it produces is packed +// into an envelope of this gateway's own. That is the whole of what the action pivot was: +// one wire dials `compact`, the other dials `generate` under a stage that folds the +// answer, and neither needs an action to travel in the record. +// +// What the simulation cannot reproduce is stated where it happens, at `summarizationTurnFor`. +// +// One thing this chain does not carry, stated rather than implied by its absence: the +// server-tool shim. It is still only an interceptor, so a compaction whose caller declared a +// server tool is summarized without the ReAct loop wrapped around it — the same gap the +// generate chain states, and for the same reason. + +import { responsesCreatedAt, wrapResponsesStatefulOutput } from './client-output.ts'; +import { completeResponsesCompaction } from './compaction-resource.ts'; +import { + buildCompactionEnvelope, + EMPTY_SUMMARY_MESSAGE, + expandShimCompactionItems, + summarizationTurnFor, + summaryTextFrom, +} from './interceptors/compact-shim.ts'; +import { syntheticEventsFromCompaction, syntheticEventsFromResult } from './items/output.ts'; +import { + beginStoredAttempt, + billedResponsesEntity, + hydrateStoredItems, + internalErrorEnvelope, + RESPONSES_STREAMED_USAGE, + responsesNarrowing, + responsesTarget, + responsesWireFor, + responsesWireRules, + type ResponsesFacts, +} from './pipeline.ts'; +import { billableUsageFromResponsesResult } from './usage.ts'; +import type { Failure } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { failover } from '../../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import { dialChatWire, type ChatWire } from '../handoff.ts'; +import { + disableReasoningOnForcedToolChoiceForResponses, + stripPromptCacheKeyForResponses, + vendorDeepSeekNormalizeForResponses, + vendorQwenNormalizeForResponses, +} from '../interceptors.ts'; +import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; +import { materializeAttempt, resolveChatCandidates, type ChatServices } from '../stages.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { renderProtocolError, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { + collectResponsesProtocolEventsToResult, + createRandomResponsesItemId, + type CanonicalResponsesPayload, + type ResponsesOutputItem, + type ResponsesStreamEvent, +} from '@floway-dev/protocols/responses'; +import { providerModelOf } from '@floway-dev/provider'; + +type R = { [P in K]: ResponsesFacts[P] }; + +/** What a compaction wire hands up. A compaction is a stream by the time it leaves either + * ending — the envelope is expanded into the events the stateful half reads — so the value + * arm this protocol's response key also admits is not one this chain ever produces. */ +type Compacted = { + [P in K]: { readonly kind: 'stream'; readonly frames: AsyncIterable } | Failure; +}; + +/** + * The outermost edge. + * + * A compaction answers with one resource, so the frames never reach a client: they are read + * here, which is also what stores each item the turn emitted under this gateway's own + * response id. The resource that comes out is `CompactResource` and not the response + * resource, so it is completed here rather than by the generate chain's egress — the fields + * the two require are different, and a compaction decorated with the twenty-odd keys a + * response resource declares would answer with a shape the spec does not give it. + */ +const emitResponsesCompaction = defineStage< + Record, + Record, + Compacted<'response.chat.responses'> & R<'response.http.headers' | 'response.chat.responses.streamedUsage'>, + R<'response.chat.responses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.responses.streamedUsage'>, + ChatServices +>({ + name: 'emitResponsesCompaction', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.chat.responses', 'response.http.headers', 'response.chat.responses.streamedUsage'], + consumes: ['response.chat.responses', 'response.http.headers'], + provides: [ + 'response.chat.responses.rendered', + 'response.http.status', + 'response.http.headers', + 'response.chat.responses.streamedUsage', + ], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.chat.responses': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move(renderProtocolError( + answer.body, + () => answer.envelope ?? { error: { message: answer.message, type: 'api_error' } }, + )), + 'response.http.status': answer.status, + }; + } + + try { + const persisted = await collectResponsesProtocolEventsToResult( + wrapResponsesStatefulOutput(answer.frames as AsyncIterable>, use.gateway), + ); + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move( + completeResponsesCompaction(persisted, responsesCreatedAt(use.gateway)) as unknown as Record, + ), + 'response.http.status': 200, + 'response.chat.responses.streamedUsage': move( + withVerdict(rest['response.chat.responses.streamedUsage'], compactionFailed(persisted)), + ), + }; + } catch (error) { + // Nothing has gone out yet, so the fault is still a status. A compaction the gateway + // could not finish — an upstream that stated no counts, a turn that could not be + // stored — is not one it can answer, and the client is told what broke. + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.responses.rendered': move(internalErrorEnvelope(error)), + 'response.http.status': 502, + 'response.chat.responses.streamedUsage': move(withVerdict(rest['response.chat.responses.streamedUsage'], true)), + }; + } + }, +}); + +/** Whether the compaction resource says the turn behind it failed. `status` is not a key the + * compaction resource declares — it survives the spread from the turn the upstream actually + * ran, on either ending — and it is authoritative: a compaction that surfaced as failed + * belongs in the error column rather than masquerading as a success. */ +const compactionFailed = (persisted: { readonly status?: string }): boolean => persisted.status === 'failed'; + +/** The accounting the epilogue settles from, with the verdict only the completed resource + * could give it. The edge is where a compaction resource comes into existence, so it is the + * one reader that can say whether the turn behind it succeeded. */ +const withVerdict = (outcome: Promise | null, failed: boolean): Promise | null => { + if (outcome === null || !failed) return outcome; + return outcome.then(read => ({ ...read, failed: true })); +}; + +/** + * The compaction wire. It asks the upstream's own `/responses` endpoint to compact, and hands + * the envelope up as the events it expands into — the same currency a generate wire hands up, + * which is what lets the stateful half above read either without knowing which ran. + */ +const callResponsesCompactUpstream = defineStage< + R<'request.chat.responses' | 'route.attempt' | 'ingress.http.headers'>, + Compacted<'response.chat.responses'> & R<'response.chat.responses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, + ChatServices +>({ + name: 'callResponsesCompactUpstream', + return: { + provides: [ + 'response.chat.responses', + RESPONSES_STREAMED_USAGE, + 'response.usage.billable', + 'response.http.headers', + ], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + + const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; + const payload = { ...asked, model: candidate.model.id }; + if (candidate.rules !== undefined) applyRulesToUpstreamResponses(payload, candidate.rules); + // Neither field belongs on this endpoint: `store` is a gateway-only snapshot hint the + // compaction endpoint rejects, and `stream` describes how an answer would be delivered + // where there is one body and no stream. + const { model: _addressed, stream: _stream, store: _store, ...body } = payload; + + let result; + try { + result = await candidate.provider.instance.callResponses( + providerModelOf(candidate), + body, + 'compact', + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.chat.responses': { status: 502, message: error instanceof Error ? error.message : String(error) }, + [RESPONSES_STREAMED_USAGE]: null, + 'response.usage.billable': [], + 'response.http.headers': [], + }); + } + + const identity = telemetryModelIdentity(candidate, result.modelKey); + if (!result.ok) { + const text = await result.response.text(); + use.log.warn('upstream refused', { status: result.response.status }); + let parsed: unknown; + try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } + return move({ + ...facts, + 'response.chat.responses': { + status: result.response.status, + message: text, + ...(parsed === undefined ? {} : { body: parsed }), + }, + [RESPONSES_STREAMED_USAGE]: null, + // An upstream that was called and reported nothing, which is a different statement + // from reporting zero. + 'response.usage.billable': [{ identity, quantities: {} }], + 'response.http.headers': [...result.response.headers], + }); + } + if (result.action !== 'compact') { + throw new Error(`callResponsesCompactUpstream: ${facts['route.attempt'].upstreamId} answered a compaction dial with a ${result.action} turn`); + } + + // This candidate answered, so it is the one a follow-up turn carrying our own state must + // come back to. + use.selectAffinity(candidate); + + const billable = [billedResponsesEntity(identity, billableUsageFromResponsesResult(result.result) ?? undefined)]; + return move({ + ...facts, + 'response.chat.responses': { kind: 'stream' as const, frames: syntheticEventsFromCompaction(result.result) }, + // A compaction states its own counts, so what it billed is known before the frames are + // read. It travels as a reading still to come rather than as one already settled, + // because the verdict that goes with it is the edge's to add. + [RESPONSES_STREAMED_USAGE]: Promise.resolve({ billable, failed: false }), + 'response.usage.billable': billable, + 'response.http.headers': [], + }); + }, +}); + +/** + * Simulates a compaction over an upstream that has no compaction wire. + * + * It rewrites the turn on the way down — the compactor's prompt at the head of the history, + * the trigger stripped, a terminal nudge appended, nothing persisted upstream — and folds the + * generated summary into an envelope of this gateway's own on the way back. Below it is an + * ordinary generate fork, which is what makes every wire generation can take a wire a + * compaction can be simulated over. + * + * A prior simulated compaction the client echoed back is expanded first, so what is summarized + * is the history it stood for rather than an opaque blob. A blob this gateway did not write + * round-trips untouched. + */ +const summarizeForCompaction = defineStage< + R<'request.chat.responses'>, + R<'request.chat.responses'>, + Compacted<'response.chat.responses'>, + Compacted<'response.chat.responses'>, + ChatServices +>({ + name: 'summarizeForCompaction', + through: { + request: { needs: ['request.chat.responses'], consumes: [], provides: ['request.chat.responses'] }, + response: { + needs: ['response.chat.responses'], + consumes: [], + provides: ['response.chat.responses'], + }, + }, + execute: async (facts, next) => { + const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; + const back = await next({ + ...facts, + 'request.chat.responses': move(summarizationTurnFor(expandShimCompactionItems(asked))), + }); + + const answer = back['response.chat.responses']; + // An upstream that refused is handed on as it came: the client learns the compaction + // failed rather than being given a silent empty envelope. + if (isFailure(answer)) return back; + + // The item lifecycle is the authority on what the turn closed; a Codex upstream states an + // `output` on its terminal that omits the message it just closed, so both are read and + // the closed items win where there are any. + const closed = new Map(); + const observed = (async function* (): AsyncIterable> { + for await (const frame of answer.frames as AsyncIterable>) { + if (frame.type === 'event' && frame.event.type === 'response.output_item.done') { + closed.set(frame.event.output_index, frame.event.item); + } + yield frame; + } + })(); + const collected = await collectResponsesProtocolEventsToResult(observed); + + const summaryText = summaryTextFrom(closed, collected.output); + if (summaryText.length === 0) { + // A summarization that closed no text produced no summary, and the blob is the whole of + // what the next turn inherits — so this is a candidate that did not do the job rather + // than a fault that ends the request, and the fork can try another. + return { ...back, 'response.chat.responses': move({ status: 502, message: EMPTY_SUMMARY_MESSAGE }) }; + } + + const synthesized = buildCompactionEnvelope(createRandomResponsesItemId('compaction'), summaryText, collected); + return { ...back, 'response.chat.responses': move({ kind: 'stream' as const, frames: syntheticEventsFromResult(synthesized) }) }; + }, +}); + +/** The generate fork, as the simulation reaches it. A summarization is an ordinary turn, so + * it is dialled on the wires an ordinary turn is dialled on. */ +const dialSummarizationWire = dialChatWire({ + source: 'request.chat.responses', + needs: ['request.chat.responses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.responses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => responsesTarget.pick(endpoints), + wire: responsesWireFor, +}); + +const compactionWire: ChatWire = compose('responsesCompactNative', [...responsesWireRules, callResponsesCompactUpstream]); +const simulationWire: ChatWire = compose('responsesCompactSimulated', [summarizeForCompaction, dialSummarizationWire]); + +/** + * Picks how this candidate's compaction is produced. + * + * An upstream whose own endpoint compacts is asked to, unless the operator asked for the + * simulation instead — which is what the `responses-compact-shim` flag is: an opt-in for a + * Responses upstream that would answer a compaction itself. Everything else is simulated, + * structurally rather than by choice: no translation carries a compaction, so a Messages or + * Chat Completions candidate has no compaction to dial. + * + * It is last, which is what earns it the right to name a target at all, and it holds no state + * across candidates: failover re-running the suffix re-runs this stage. + */ +const dialResponsesCompaction = defineStage< + R<'route.attempt'> & Record, + Record, + Record, + Record, + ChatServices +>({ + name: 'dialResponsesCompaction', + into: { + request: { + needs: ['route.attempt', 'request.chat.responses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + consumes: [], + provides: [], + }, + // Nothing is read on the way back — a wire hands up this family's own keys and they ride + // through — but this is where they enter the chain, so this is the stage that provides + // them and the runner checks that the wire delivered. + response: { + needs: [], + consumes: [], + provides: [ + 'response.chat.responses', + RESPONSES_STREAMED_USAGE, + 'response.usage.billable', + 'response.http.headers', + ], + }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const compacts = responsesTarget.pick(candidate.model.endpoints) === 'responses' + && !facts['route.attempt'].flags.includes('responses-compact-shim'); + use.log.debug('compacting', { upstream: facts['route.attempt'].upstreamId, wire: compacts ? 'compact' : 'simulated' }); + return await next(facts, compacts ? compactionWire : simulationWire); + }, +}); + +export type ResponsesCompactEntry = R< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.responses' | 'serve.model' +>; + +export type ResponsesCompactExit = R< + 'response.chat.responses.rendered' | 'response.chat.responses.streamedUsage' + | 'response.http.status' | 'response.http.headers' +>; + +export const responsesCompactPipeline = (payload: CanonicalResponsesPayload): Pipeline => { + // One cell per run, written by the stage directly above the one that reads it — the same + // hand-across the generate chain makes, for the same reason: the resolver takes its + // narrowing at assembly, before any fact exists. + let prepared = payload; + return compose('responsesCompact', [ + emitResponsesCompaction, + writeSettlement( + handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), + handedUp => (handedUp as { 'response.chat.responses.streamedUsage'?: unknown })['response.chat.responses.streamedUsage'] !== null, + ), + hydrateStoredItems(payload, hydrated => { prepared = hydrated; }), + resolveChatCandidates(responsesNarrowing(() => prepared)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), + owns: [], + }), + materializeAttempt('request.chat.responses'), + beginStoredAttempt, + disableReasoningOnForcedToolChoiceForResponses, + stripPromptCacheKeyForResponses, + vendorDeepSeekNormalizeForResponses, + vendorQwenNormalizeForResponses, + dialResponsesCompaction, + ]); +}; diff --git a/packages/gateway/src/data-plane/chat/responses/http.ts b/packages/gateway/src/data-plane/chat/responses/http.ts index c25a6a3f0e..59bc933b39 100644 --- a/packages/gateway/src/data-plane/chat/responses/http.ts +++ b/packages/gateway/src/data-plane/chat/responses/http.ts @@ -1,91 +1,25 @@ -// POST /v1/responses, served through the pipeline, and POST /v1/responses/compact beside it -// — a second operation over this protocol rather than another wire under its chain, so it -// still goes through `responsesServe.compact`. +// POST /v1/responses and POST /v1/responses/compact, both served through their own pipeline. +// Generation is one chain; compaction is a second operation over the same protocol and is a +// chain of its own rather than another wire under the first. // -// The generate entry is a prologue and an epilogue around `responsesServePipeline`: read -// what the client sent, hand it over, and turn what the run answered with into a response. -// Everything between is stages, the stored-items membrane included — which is why a -// continuation that does not resolve is answered by the chain here and still caught as a -// throw on the compact route, which has not moved. +// Each entry is a prologue and an epilogue around that chain: read what the client sent, hand +// it over, and turn what the run answered with into a response. Everything between is stages, +// the stored-items membrane included — which is why a continuation that does not resolve is +// answered by the chain on both routes rather than caught as a throw on either. -import { responsesInputErrorResult } from './errors.ts'; +import { responsesCompactPipeline } from './compact.ts'; import { createResponsesHttpStore } from './items/store.ts'; import { responsesServePipeline } from './pipeline.ts'; -import { respondResponses, respondResponsesFailure } from './respond.ts'; -import { PreviousResponseNotFoundError } from './serve-prep.ts'; -import { responsesServe } from './serve.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; -import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; -import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; +import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; -import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerModelsUnavailableResponse } from '../shared/upstream-models-error.ts'; import { move } from '@floway-dev/pipeline'; import type { CanonicalResponsesPayload, ResponsesRequestPayload } from '@floway-dev/protocols/responses'; -import { internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; -// OpenAI's verbatim previous_response_not_found envelope. Codex compares this -// body byte-for-byte against upstream — see the cross-references on -// `PreviousResponseNotFoundError` in serve-prep.ts. -const previousResponseNotFoundResponse = (id: string): Response => - Response.json( - { - error: { - message: `Previous response with id '${id}' not found.`, - type: 'invalid_request_error', - param: 'previous_response_id', - code: 'previous_response_not_found', - }, - }, - { status: 400 }, - ); - -// Surfaces a pre-stream throw (malformed JSON body, an interceptor crash, -// etc.) as a Responses-shaped 502 with the same internal-error envelope the -// in-flow `internal-error` ExecuteResult produces. A -// `ProviderModelsUnavailableError` carrying an upstream HTTP body relays -// that body verbatim — the upstream's `/models` 401 IS the diagnostic. The -// caller passes its outer `ctx` when one was already constructed (so the -// dump row preserves the model attribution the request-time -// `requestedModel` stamped, and the throwing-candidate telemetry stamped -// in serve.ts survives onto the error row); a fresh ctx is minted only -// for pre-parse failures where no payload was available to read model from. -const respondWithInternalError = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - const verbatim = providerModelsUnavailableResponse(error); - if (verbatim !== null) return verbatim; - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const result = internalErrorResult(502, toInternalDebugError(error), effectiveCtx.attempt.telemetry); - const response = respondResponsesFailure(result, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); -}; - -// Pre-stream throw dispatcher. `PreviousResponseNotFoundError` and the -// translator-input case render protocol-shaped 400s; anything else falls -// through to the internal-error 502 path. -const respondToThrow = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - if (error instanceof PreviousResponseNotFoundError) { - const response = previousResponseNotFoundResponse(error.previousResponseId); - ctx?.dump?.error('gateway'); - return ctx ? finalizeGatewayResponse(ctx, response) : response; - } - if (error instanceof TranslatorInputError) { - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const response = respondResponsesFailure(responsesInputErrorResult(error, effectiveCtx.attempt.telemetry), effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); - } - return await respondWithInternalError(c, error, requestBody, ctx); -}; - -const parsePayload = (bytes: Uint8Array): CanonicalResponsesPayload => - canonicalizeResponsesPayload(JSON.parse(new TextDecoder().decode(bytes)) as ResponsesRequestPayload); - -/** The same read, as a value rather than a throw, because a pipelined entry decides what to - * do about a body it could not read before it opens a run rather than after one unwound. +/** The read, as a value rather than a throw, because a pipelined entry decides what to do + * about a body it could not read before it opens a run rather than after one unwound. * * What the reader rejected travels with the rejection. This protocol's envelope names the * field at fault and the code for the condition, and those are statements only the reader @@ -95,7 +29,7 @@ const readRequest = (bytes: Uint8Array): | { type: 'ok'; payload: CanonicalResponsesPayload } | { type: 'invalid'; message: string; param?: string; code?: string } => { try { - return { type: 'ok', payload: parsePayload(bytes) }; + return { type: 'ok', payload: canonicalizeResponsesPayload(JSON.parse(new TextDecoder().decode(bytes)) as ResponsesRequestPayload) }; } catch (error) { if (!(error instanceof TranslatorInputError)) { return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; @@ -125,17 +59,19 @@ const invalidRequestResponse = (invalid: { message: string; param?: string; code { status: 400 }, ); +/** A request the gateway refused before it reached a pipeline: there is no model to resolve + * and no attempt to make, so there is nothing for a run to record beyond the refusal. */ +const refuse = (c: AuthedContext, ingress: Ingress, invalid: { message: string; param?: string; code?: string }): Response => { + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse(refused.gateway, invalidRequestResponse(invalid)); +}; + export const responsesHttp = { generate: async (c: AuthedContext): Promise => { const ingress = await readIngress(c); const request = readRequest(ingress.body.bytes); - if (request.type === 'invalid') { - // A request the gateway could not read never reaches a pipeline: there is no model to - // resolve and no attempt to make, so there is nothing for a run to record. - const refused = openPrologue(c, ingress, { wantsStream: false }); - refused.gateway.dump?.error('gateway'); - return finalizeGatewayResponse(refused.gateway, invalidRequestResponse(request)); - } + if (request.type === 'invalid') return refuse(c, ingress, request); const { payload } = request; const wantsStream = payload.stream === true; @@ -166,41 +102,31 @@ export const responsesHttp = { }, compact: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = parsePayload(requestBody.bytes); - ctx = createChatGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, (apiKey, requestStartedAt) => createResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined)); - const result = await responsesServe.compact({ payload, ctx, headers: inboundHeaders(c) }); - if (result.type === 'result') { - // Compact drains the upstream stream into a single compaction - // resource with no per-token stamps; recordPerformance therefore - // lands in the neutral bucket (request counted, no TTFT/TPOT sample). - // `status` is not a `CompactResource` key — it survives the spread - // from the upstream turn — and it is authoritative for failure: a - // compact that surfaced as `response.failed` must be recorded as such - // so it shows up in the error column instead of masquerading as a - // success. - const failed = result.result.status === 'failed'; - if (failed) { - ctx.dump?.failed('compact envelope status=failed'); - } else { - ctx.dump?.success(result.modelIdentity, result.usage); - } - settle( - ctx, - result.performance, - result.modelIdentity, - result.usage, - failed, - ); - const compactResponse = Response.json(result.result); - return finalizeGatewayResponse(ctx, compactResponse); - } - const response = await respondResponses(c, result, false, ctx, payload); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); - } + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') return refuse(c, ingress, request); + + const { payload } = request; + const prologue = openChatPrologue(c, ingress, { + wantsStream: false, + model: payload.model, + storeFactory: (apiKey, requestStartedAt) => createResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined), + }); + + return await serveThrough( + c, + prologue, + responsesCompactPipeline(payload), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'responses', + 'request.chat.responses': payload, + 'serve.model': payload.model, + }) as never, + // A compaction is one resource however the turn went, so there is never a stream to + // write: the frames it came as were read into that resource before the run answered. + facts => ({ body: JSON.stringify(facts['response.chat.responses.rendered']), contentType: 'application/json' }), + facts => facts['response.chat.responses.streamedUsage'], + ); }, }; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts index c0ced53e0d..4464cc2367 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts @@ -189,7 +189,7 @@ type ChainRun = () => Promise> // assistant message it just closed. A turn that closed nothing falls back to // the terminal, as the client-facing egress does. // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L237 -const summaryTextFrom = (closed: Map, stated: readonly ResponsesOutputItem[]): string => { +export const summaryTextFrom = (closed: Map, stated: readonly ResponsesOutputItem[]): string => { const items = closed.size === 0 ? stated : [...closed].sort(([left], [right]) => left - right).map(([, item]) => item); @@ -203,7 +203,7 @@ const summaryTextFrom = (closed: Map, stated: reado return parts.join(''); }; -const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: ResponsesResult): ResponsesResult => { +export const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: ResponsesResult): ResponsesResult => { // The prefix lives inside the blob so it round-trips atomically with the // summary — a downstream LLM sees `${SUMMARY_PREFIX}\n${summaryText}` in // one message and reads it as "another LLM's handoff", not as the human @@ -243,12 +243,15 @@ const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: R }; }; -const simulateCompaction = async (ctx: ResponsesInvocation, gatewayCtx: ChatGatewayCtx, run: ChainRun): Promise>> => { - const originalPayload = ctx.payload; - +// The turn a compaction is simulated with: the compactor's own prompt, the +// history that is being compacted, and a nudge to produce the summary now. +// Exported because the pipeline's compaction chain sends the same turn — one +// definition is what keeps the simulated compaction identical whichever entry +// asked for it. +export const summarizationTurnFor = (payload: CanonicalResponsesPayload): CanonicalResponsesPayload => { // Strip compaction_trigger so the upstream sees a plain generate turn // against SUMMARIZATION_PROMPT. - const historyItems = originalPayload.input.filter(item => item.type !== 'compaction_trigger'); + const historyItems = payload.input.filter(item => item.type !== 'compaction_trigger'); // Anthropic Messages rejects assistant prefill — when the translated // conversation ends on an assistant message, the upstream returns 400 @@ -303,15 +306,22 @@ const simulateCompaction = async (ctx: ResponsesInvocation, gatewayCtx: ChatGate role: 'system', content: [{ type: 'input_text', text: SUMMARIZATION_PROMPT }], }; - const inputForSummarization = [compactorSystemMessage, ...historyItems, terminalUserMessage]; - - ctx.payload = { - ...originalPayload, - input: inputForSummarization, + return { + ...payload, + input: [compactorSystemMessage, ...historyItems, terminalUserMessage], // Do not persist the ephemeral summarization turn in the upstream's // conversation history. store: false, }; +}; + +// A summarization that closed no assistant text produced no summary, and a +// compaction blob is the whole of what the next turn inherits — so an empty one +// silently discards the conversation. +export const EMPTY_SUMMARY_MESSAGE = 'Responses compact shim: the summarization turn closed no assistant text to summarize'; + +const simulateCompaction = async (ctx: ResponsesInvocation, gatewayCtx: ChatGatewayCtx, run: ChainRun): Promise>> => { + ctx.payload = summarizationTurnFor(ctx.payload); // Pivot the action so the inner dispatch routes to the upstream's // generate wire instead of its compact wire. The mutation is one-way: // the project's interceptor convention is that every `ctx.*` write @@ -340,11 +350,7 @@ const simulateCompaction = async (ctx: ResponsesInvocation, gatewayCtx: ChatGate const collected = await collectResponsesProtocolEventsToResult(observed); const summaryText = summaryTextFrom(closedItems, collected.output); - // A compaction blob is the whole of what the next turn inherits, so an empty - // one silently discards the conversation. - if (summaryText.length === 0) { - throw new Error('Responses compact shim: the summarization turn closed no assistant text to summarize'); - } + if (summaryText.length === 0) throw new Error(EMPTY_SUMMARY_MESSAGE); const cmpId = createRandomResponsesItemId('compaction'); const synthesized = buildCompactionEnvelope(cmpId, summaryText, collected); diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index ae59c07c9d..d66cacb68c 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -25,9 +25,11 @@ // absence: // // - `/v1/responses/compact`. A second operation over this protocol rather than another -// wire under this pipeline, so it stays on `responsesServe.compact`. The dial here asks -// for `generate`; what the ending answers with is the branch the provider says it ran, -// which is why the envelope a compaction is arrives somewhere rather than nowhere. +// wire under this pipeline, so it is a chain of its own in `compact.ts` — which reuses +// this one's membrane, its narrowing and its wires, because a compaction routes and is +// rewritten exactly as a turn is. The dial here asks for `generate`; what the ending +// answers with is the branch the provider says it ran, which is why the envelope a +// compaction is arrives somewhere rather than nowhere. // - this family's remaining interceptors, the server-tool shims among them. Still only in // the interceptor form, so the array between the materialized payload and the fork is // short rather than complete — and nothing in a pipelined turn writes to the store's @@ -228,7 +230,7 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre /** What a fault the gateway is answerable for looks like on this protocol, with the stack * that says where it happened: this is the gateway's own failure and not an upstream's, so * the body is a diagnostic rather than something a client is meant to parse. */ -const internalErrorEnvelope = (error: unknown): Record => { +export const internalErrorEnvelope = (error: unknown): Record => { const debug = toInternalDebugError(error); return { error: { @@ -410,7 +412,7 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< ...facts, 'response.chat.responses': { kind: 'value' as const, body: result.result }, [streamedUsage]: null, - 'response.usage.billable': [billed(identity, billableUsageFromResponsesResult(result.result) ?? undefined)], + 'response.usage.billable': [billedResponsesEntity(identity, billableUsageFromResponsesResult(result.result) ?? undefined)], 'response.http.headers': [], }); } @@ -461,31 +463,39 @@ const normalizeAssistantContentForResponses = defineStage< }); /** - * The Responses wire, as the chain that dials it. + * What every turn this wire sends is subject to, whichever operation asked for it. * - * Every source protocol that reaches an upstream over this endpoint runs this, whether the + * Every source protocol that reaches an upstream over this endpoint runs these, whether the * client spoke Responses or a handoff arrived here — which is what makes the three rules - * above the dial belong here. The role rewrite and the assistant-content rewrite both state - * what an upstream's Responses endpoint accepts; the cache-bucket fold speaks about the usage - * *this* wire reports and about the flag that describes it, and a translator emits the - * canonical form, which is the one case the fold has nothing to do with. + * belong to the wire. The role rewrite and the assistant-content rewrite both state what an + * upstream's Responses endpoint accepts; the cache-bucket fold speaks about the usage *this* + * wire reports and about the flag that describes it, and a translator emits the canonical + * form, which is the one case the fold has nothing to do with. + * + * They are named apart from the dial because a compaction is dialled differently and is + * subject to the same three. */ -export const responsesWire = (streamedUsage: string): readonly Stage[] => [ +export const responsesWireRules: readonly Stage[] = [ normalizeAssistantContentForResponses, applyRoleCompatibilityToResponses, normalizeExclusiveCachedTokensForResponses, +]; + +/** The Responses wire, as the chain that dials it. */ +export const responsesWire = (streamedUsage: string): readonly Stage[] => [ + ...responsesWireRules, callResponsesUpstream(streamedUsage), ]; /** This family's own reading, which every wire under it hands up. */ -const STREAMED_USAGE = 'response.chat.responses.streamedUsage'; +export const RESPONSES_STREAMED_USAGE = 'response.chat.responses.streamedUsage'; /** The three wires `/v1/responses` can be served on. Its own is the bare wire; each translated * one is a handoff and then the target protocol's own wire. */ -const responsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { +export const responsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { switch (target) { case 'responses': - return compose('responsesNative', responsesWire(STREAMED_USAGE)); + return compose('responsesNative', responsesWire(RESPONSES_STREAMED_USAGE)); case 'messages': return compose('responsesViaMessages', [ handOff({ @@ -497,7 +507,7 @@ const responsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), }), }), - ...messagesWire(STREAMED_USAGE), + ...messagesWire(RESPONSES_STREAMED_USAGE), ]); case 'chat-completions': return compose('responsesViaChatCompletions', [ @@ -506,7 +516,7 @@ const responsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, trip: async payload => await translateResponsesViaChatCompletions(payload, { model: candidate.model.id }), }), - ...chatCompletionsWire(STREAMED_USAGE), + ...chatCompletionsWire(RESPONSES_STREAMED_USAGE), ]); } }; @@ -559,7 +569,7 @@ const meterResponses = ( // Reached however the frames ended — the terminal event, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle({ billable: [billed(identity, reported)], failed: !sawTerminal }); + settle({ billable: [billedResponsesEntity(identity, reported)], failed: !sawTerminal }); } })(); return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; @@ -573,7 +583,7 @@ const meterResponses = ( * because on this protocol it is not a quantity but a rate selector — `service_tier` states * the tier the turn was actually served at, and that is the pricing entry it is billed * under. */ -const billed = (identity: TelemetryModelIdentity, usage: BillableUsage | undefined): BillableEntity => { +export const billedResponsesEntity = (identity: TelemetryModelIdentity, usage: BillableUsage | undefined): BillableEntity => { if (usage === undefined) return { identity, quantities: {} }; const measurement = tokenUsageMeasurement(tokenUsageFromBillableUsage(usage)); return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; @@ -588,7 +598,7 @@ const billed = (identity: TelemetryModelIdentity, usage: BillableUsage | undefin * client never sent, and a turn is pinned by what its items carry. The narrowing is built * at assembly, before any fact exists, so the membrane hands the prepared payload across * through the run's own cell rather than through the record. */ -const narrowing = (prepared: () => CanonicalResponsesPayload): ChatNarrowing> => ({ +export const responsesNarrowing = (prepared: () => CanonicalResponsesPayload): ChatNarrowing> => ({ canServe: candidate => responsesTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeResponsesAffinity(prepared(), gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /responses endpoint.`, @@ -645,7 +655,7 @@ const refuseStoredItems = ( * nowhere to be routed to. Both of its refusals are answers it already holds, which is why * it carries the `return` trait alongside `through`. */ -const hydrateStoredItems = ( +export const hydrateStoredItems = ( client: CanonicalResponsesPayload, prepared: (payload: CanonicalResponsesPayload) => void, ) => defineStage< @@ -731,7 +741,7 @@ const hydrateStoredItems = ( * an interceptor — so today this only puts back the state the stored rows already carried, * which is what lets an item this turn re-emits be stored with it intact. */ -const beginStoredAttempt = defineStage< +export const beginStoredAttempt = defineStage< R<'request.chat.responses.privatePayloads'>, R<'request.chat.responses.privatePayloads'>, Record, @@ -777,7 +787,7 @@ export const responsesServePipeline = ( handedUp => (handedUp as { 'response.chat.responses.streamedUsage'?: unknown })['response.chat.responses.streamedUsage'] !== null, ), hydrateStoredItems(payload, hydrated => { prepared = hydrated; }), - resolveChatCandidates(narrowing(() => prepared)), + resolveChatCandidates(responsesNarrowing(() => prepared)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), owns: [], @@ -791,7 +801,7 @@ export const responsesServePipeline = ( dialChatWire({ source: 'request.chat.responses', needs: ['request.chat.responses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.responses', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + provides: ['response.chat.responses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], pick: endpoints => responsesTarget.pick(endpoints), wire: responsesWireFor, }), From f689ffaa01ae4b4f0596069b32ba10badb699921 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 10:19:40 +0800 Subject: [PATCH 048/156] refactor(chat): delete the onion the pipeline replaced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every chat entry — four families' generate, both count-token operations, the compaction, and the WebSocket transport — serves through a chain now, so the serve/attempt/respond layers below them have no caller. Seventy-five files go, including all four attempt modules, the three interceptor registries, and every rule that became a stage. iterate-candidates.ts goes with them: the fork is a stage, and the loop it replaced was the last thing holding that shape. What stays is what a chain still reaches: the Claude Code probe's recognition and frames, the web-search request shaping the count-token chain runs, the compaction shim's summarization, SourceStreamState for the socket, and the server-tool shim — the one piece whose behaviour has not been ported, kept so that porting it does not start from nothing. --- .../chat/chat-completions/attempt_test.ts | 318 -------- .../chat/chat-completions/errors_test.ts | 44 -- .../apply-role-compatibility_test.ts | 121 --- ...le-reasoning-on-forced-tool-choice_test.ts | 77 -- .../include-usage-stream-options_test.ts | 57 -- .../normalize-exclusive-cached-tokens_test.ts | 136 ---- .../interceptors/normalize-usage_test.ts | 149 ---- .../strip-prompt-cache-key_test.ts | 54 -- .../vendor-deepseek-normalize_test.ts | 393 --------- .../vendor-kimi-normalize_test.ts | 134 ---- .../vendor-qwen-normalize_test.ts | 76 -- .../chat/chat-completions/respond_test.ts | 66 -- .../chat/chat-completions/serve_test.ts | 398 ---------- .../data-plane/chat/gemini/attempt_test.ts | 274 ------- .../strip-safety-settings_test.ts | 43 - .../strip-unsupported-part-fields_test.ts | 128 --- .../strip-unsupported-tools_test.ts | 87 -- .../suppress-thought-parts_test.ts | 151 ---- .../data-plane/chat/gemini/serve_test.ts | 479 ----------- .../data-plane/chat/messages/attempt_test.ts | 529 ------------- .../apply-role-compatibility_test.ts | 71 -- ...le-reasoning-on-forced-tool-choice_test.ts | 93 --- .../strip-billing-attribution_test.ts | 154 ---- .../data-plane/chat/messages/respond_test.ts | 127 --- .../data-plane/chat/messages/serve_test.ts | 743 ------------------ .../data-plane/chat/responses/attempt_test.ts | 664 ---------------- .../apply-role-compatibility_test.ts | 124 --- ...le-reasoning-on-forced-tool-choice_test.ts | 76 -- .../normalize-exclusive-cached-tokens_test.ts | 128 --- .../strip-prompt-cache-key_test.ts | 71 -- .../vendor-deepseek-normalize_test.ts | 67 -- .../data-plane/chat/responses/serve_test.ts | 652 --------------- .../data-plane/pipeline/assembly_test.ts | 6 + .../shared/iterate-candidates_test.ts | 124 --- .../chat/chat-completions/attempt.ts | 79 -- .../chat/chat-completions/errors.ts | 32 - .../interceptors/apply-role-compatibility.ts | 30 - ...disable-reasoning-on-forced-tool-choice.ts | 24 - .../include-usage-stream-options.ts | 20 - .../chat-completions/interceptors/index.ts | 59 -- .../normalize-exclusive-cached-tokens.ts | 97 --- .../interceptors/normalize-usage.ts | 44 -- .../interceptors/strip-prompt-cache-key.ts | 16 - .../chat-completions/interceptors/types.ts | 13 - .../interceptors/vendor-deepseek-normalize.ts | 143 ---- .../interceptors/vendor-kimi-normalize.ts | 59 -- .../interceptors/vendor-qwen-normalize.ts | 30 - .../chat/chat-completions/respond.ts | 130 --- .../data-plane/chat/chat-completions/serve.ts | 55 -- .../src/data-plane/chat/gemini/attempt.ts | 170 ---- .../chat/gemini/interceptors/index.ts | 29 - .../interceptors/strip-safety-settings.ts | 11 - .../strip-unsupported-part-fields.ts | 29 - .../interceptors/strip-unsupported-tools.ts | 38 - .../interceptors/suppress-thought-parts.ts | 49 -- .../chat/gemini/interceptors/types.ts | 23 - .../src/data-plane/chat/gemini/serve.ts | 84 -- .../src/data-plane/chat/messages/attempt.ts | 121 --- .../interceptors/apply-role-compatibility.ts | 15 - ...disable-reasoning-on-forced-tool-choice.ts | 32 - .../chat/messages/interceptors/index.ts | 59 -- .../interceptors/strip-billing-attribution.ts | 42 - .../src/data-plane/chat/messages/respond.ts | 132 ---- .../src/data-plane/chat/messages/serve.ts | 85 -- .../src/data-plane/chat/responses/attempt.ts | 244 ------ .../interceptors/apply-role-compatibility.ts | 35 - ...disable-reasoning-on-forced-tool-choice.ts | 25 - .../chat/responses/interceptors/index.ts | 60 -- .../normalize-exclusive-cached-tokens.ts | 76 -- .../interceptors/strip-prompt-cache-key.ts | 15 - .../interceptors/vendor-deepseek-normalize.ts | 44 -- .../src/data-plane/chat/responses/respond.ts | 183 ----- .../data-plane/chat/responses/serve-prep.ts | 73 +- .../src/data-plane/chat/responses/serve.ts | 93 --- .../chat/shared/upstream-models-error.ts | 13 - .../src/data-plane/embeddings/usage.ts | 10 - .../data-plane/shared/iterate-candidates.ts | 78 -- 77 files changed, 7 insertions(+), 9506 deletions(-) delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/attempt_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/errors_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/apply-role-compatibility_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/include-usage-stream-options_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/normalize-usage_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/respond_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/chat-completions/serve_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/gemini/attempt_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-safety-settings_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-unsupported-tools_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/gemini/interceptors/suppress-thought-parts_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/gemini/serve_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/messages/attempt_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/messages/interceptors/apply-role-compatibility_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/messages/interceptors/strip-billing-attribution_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/messages/respond_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/messages/serve_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/responses/attempt_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/responses/interceptors/apply-role-compatibility_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/responses/interceptors/strip-prompt-cache-key_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/responses/interceptors/vendor-deepseek-normalize_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/responses/serve_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/attempt.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/errors.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/apply-role-compatibility.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/include-usage-stream-options.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/index.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/normalize-usage.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/types.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/respond.ts delete mode 100644 packages/gateway/src/data-plane/chat/chat-completions/serve.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/attempt.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/interceptors/index.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/interceptors/strip-safety-settings.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/interceptors/strip-unsupported-tools.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/interceptors/suppress-thought-parts.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/interceptors/types.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/serve.ts delete mode 100644 packages/gateway/src/data-plane/chat/messages/attempt.ts delete mode 100644 packages/gateway/src/data-plane/chat/messages/interceptors/apply-role-compatibility.ts delete mode 100644 packages/gateway/src/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice.ts delete mode 100644 packages/gateway/src/data-plane/chat/messages/interceptors/index.ts delete mode 100644 packages/gateway/src/data-plane/chat/messages/interceptors/strip-billing-attribution.ts delete mode 100644 packages/gateway/src/data-plane/chat/messages/respond.ts delete mode 100644 packages/gateway/src/data-plane/chat/messages/serve.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/attempt.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/apply-role-compatibility.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/index.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/strip-prompt-cache-key.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/vendor-deepseek-normalize.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/respond.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/serve.ts delete mode 100644 packages/gateway/src/data-plane/chat/shared/upstream-models-error.ts delete mode 100644 packages/gateway/src/data-plane/embeddings/usage.ts delete mode 100644 packages/gateway/src/data-plane/shared/iterate-candidates.ts diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/attempt_test.ts deleted file mode 100644 index 708b21cea1..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/attempt_test.ts +++ /dev/null @@ -1,318 +0,0 @@ -import { test, vi } from 'vitest'; - -import { chatCompletionsAttempt } from '../../../../src/data-plane/chat/chat-completions/attempt.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import { initExternalResourceFetcher } from '@floway-dev/platform'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ResponsesPayload, ResponsesResult } from '@floway-dev/protocols/responses'; -import { type MessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderResponsesResult, type ProviderStreamResult, type ResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import type { FlagId } from '@floway-dev/provider/flags'; -import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; - -const API_KEY_ID = 'key_chat_completions_attempt_test'; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): ChatCompletionsPayload => ({ - model: 'test-model', - messages: [{ role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeChatCompletionsEvents = (): readonly ChatCompletionsStreamEvent[] => [ - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }], - }, - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }, - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, -]; - -const makeMessagesEvents = (): readonly MessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id: 'msg_cc_via_m', type: 'message', role: 'assistant', content: [], - model: 'test-model', stop_reason: null, stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - endpoints?: ModelEndpoints; - callChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: MessagesUpstreamCallOptions) => Promise>; - callResponses?: (model: unknown, body: unknown, action: ResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - enabledFlags?: ReadonlySet; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const endpoints = overrides.endpoints ?? { chatCompletions: {}, responses: {}, messages: {} }; - const provider = stubProvider({ - callChatCompletions: overrides.callChatCompletions, - callMessages: overrides.callMessages, - callResponses: overrides.callResponses, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, - }, - model: stubInternalModel({ - endpoints, - providerModels: { - [upstream]: stubProviderModel({ endpoints, enabledFlags: new Set(overrides.enabledFlags ?? []) }), - }, - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -test('generate native chat-completions target calls provider.callChatCompletions', async () => { - installRepo(); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'k', headers: new Headers(), - })); - const result = await chatCompletionsAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callChatCompletions }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callChatCompletions.mock.calls.length, 1); -}); - -test('generate native target applies role compatibility flags in target-chain order', async () => { - installRepo(); - let observedBody: Omit | undefined; - const callChatCompletions = vi.fn(async (_model, body): Promise> => { - observedBody = body as Omit; - return { - ok: true, - events: makeProtocolFrames(makeChatCompletionsEvents()), - modelKey: 'k', - headers: new Headers(), - }; - }); - const result = await chatCompletionsAttempt.generate({ - payload: makePayload({ - messages: [ - { role: 'system', content: 'base instructions' }, - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callChatCompletions, - endpoints: { chatCompletions: {} }, - enabledFlags: new Set([ - 'rewrite-developer-to-system', - 'rewrite-mid-conv-system-to-user', - 'rewrite-system-to-developer', - ]), - }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(observedBody?.messages, [ - { role: 'system', content: 'base instructions' }, - { role: 'user', content: 'hello' }, - { role: 'user', content: 'inline instructions' }, - ]); -}); - -test('generate translates through the Messages target when only that endpoint is exposed', async () => { - installRepo(); - let anthropicBeta: readonly string[] | undefined; - const callMessages = vi.fn(async (_model, _body, _signal, opts): Promise> => { - anthropicBeta = opts?.anthropicBeta; - return { ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers() }; - }); - const result = await chatCompletionsAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessages, endpoints: { messages: {} } }), - headers: new Headers({ 'anthropic-beta': 'must-not-cross-source-protocols' }), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callMessages.mock.calls.length, 1); - assertEquals(anthropicBeta, []); -}); - -test('generate injects the platform external-image loader into Chat-to-Messages translation', async () => { - installRepo(); - initExternalResourceFetcher(url => { - assertEquals(url.href, 'https://example.com/image.png'); - return Promise.resolve(new Response(Uint8Array.of(1, 2, 3), { headers: { 'content-type': 'image/png' } })); - }); - let observedBody: Omit | undefined; - const callMessages = vi.fn(async (_model, body): Promise> => { - observedBody = body as Omit; - return { ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers() }; - }); - const result = await chatCompletionsAttempt.generate({ - payload: makePayload({ - messages: [{ - role: 'user', - content: [{ type: 'image_url', image_url: { url: 'https://example.com/image.png' } }], - }], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessages, endpoints: { messages: {} } }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - const message = observedBody?.messages[0]; - assert(message?.role === 'user' && Array.isArray(message.content)); - const image = message.content.find(block => block.type === 'image'); - assert(image?.type === 'image'); - assertEquals(image.source, { type: 'base64', media_type: 'image/png', data: 'AQID' }); -}); - -test('generate translates through the Responses target when only that endpoint is exposed', async () => { - installRepo(); - const respResp: ResponsesResult = { - id: 'resp_x', object: 'response', model: 'test-model', status: 'completed', - output: [{ - type: 'message', id: 'msg_resp', role: 'assistant', status: 'completed', - content: [{ type: 'output_text', text: 'hi', annotations: [] }], - }], - output_text: 'hi', error: null, incomplete_details: null, - }; - const callResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProtocolFrames([{ type: 'response.completed', sequence_number: 0, response: respResp }]), - modelKey: 'k', - headers: new Headers(), - })); - const result = await chatCompletionsAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callResponses, endpoints: { responses: {} } }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callResponses.mock.calls.length, 1); -}); - -test('generate preserves translated instructions before rewriting inline system messages', async () => { - installRepo(); - const observedBodies: Omit[] = []; - const callResponses = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { - action: 'generate', - ok: true, - events: makeProtocolFrames([{ - type: 'response.completed', sequence_number: 0, response: { - id: 'resp_x', object: 'response', model: 'test-model', status: 'completed', - output: [], output_text: '', error: null, incomplete_details: null, - }, - }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - - const result = await chatCompletionsAttempt.generate({ - payload: makePayload({ - messages: [ - { role: 'system', content: 'base instructions' }, - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callResponses, - endpoints: { responses: {} }, - enabledFlags: new Set(['rewrite-system-to-developer']), - }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - const observedBody = observedBodies[0]; - if (!observedBody) throw new Error('expected observed Responses body'); - assertEquals(observedBody.instructions, 'base instructions'); - const input = observedBody.input; - if (!Array.isArray(input)) throw new Error('expected Responses input array'); - assertEquals(input[0], { type: 'message', role: 'user', content: 'hello' }); - assertEquals(input[1], { type: 'message', role: 'developer', content: 'inline instructions' }); -}); - -test('generate propagates upstream response headers onto the EventResult so respond can forward them', async () => { - installRepo(); - const upstreamHeaders = new Headers({ - 'anthropic-ratelimit-unified-status': 'allowed', - 'cf-ray': 'cf_ray_cc', - }); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'k', headers: upstreamHeaders, - })); - const result = await chatCompletionsAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callChatCompletions }), - headers: new Headers(), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - assertEquals(result.headers?.get('anthropic-ratelimit-unified-status'), 'allowed'); - assertEquals(result.headers?.get('cf-ray'), 'cf_ray_cc'); - await collectEvents(result.events); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/errors_test.ts deleted file mode 100644 index 6cd44bb6f6..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/errors_test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { test } from 'vitest'; - -import { translatorInputErrorResult } from '../../../../src/data-plane/chat/chat-completions/errors.ts'; -import type { ApiErrorResult } from '@floway-dev/provider'; -import { assertEquals } from '@floway-dev/test-utils'; -import { TranslatorInputError } from '@floway-dev/translate'; - -const apiErrorOf = (result: ReturnType): ApiErrorResult => result as ApiErrorResult; -const bodyOf = (result: ReturnType): unknown => - JSON.parse(new TextDecoder().decode(apiErrorOf(result).body)); - -test('translatorInputErrorResult renders an OpenAI 400 invalid_request_error envelope with default `messages` param', () => { - const result = translatorInputErrorResult( - new TranslatorInputError("Missing required field 'tool_call_id' on a 'tool' role message."), - ); - const apiError = apiErrorOf(result); - - assertEquals(apiError.type, 'api-error'); - assertEquals(apiError.source, 'gateway'); - assertEquals(apiError.status, 400); - assertEquals(bodyOf(result), { - error: { - message: "Missing required field 'tool_call_id' on a 'tool' role message.", - type: 'invalid_request_error', - param: 'messages', - code: null, - }, - }); -}); - -test('translatorInputErrorResult honors an explicit param from the translator', () => { - const result = translatorInputErrorResult( - new TranslatorInputError('content part not supported', { param: 'messages[2].content[1]' }), - ); - - assertEquals(bodyOf(result), { - error: { - message: 'content part not supported', - type: 'invalid_request_error', - param: 'messages[2].content[1]', - code: null, - }, - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/apply-role-compatibility_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/apply-role-compatibility_test.ts deleted file mode 100644 index 7c8b8235c3..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/apply-role-compatibility_test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { test } from 'vitest'; - -import { withRoleCompatibilityApplied } from '../../../../../src/data-plane/chat/chat-completions/interceptors/apply-role-compatibility.ts'; -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsMessage, ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const gatewayCtx = mockChatGatewayCtx(); -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const applyRoles = async ( - messages: ChatCompletionsMessage[], - enabledFlags: ReadonlySet, - targetApi: ChatCompletionsInvocation['targetApi'] = 'chat-completions', -): Promise => { - const payload: ChatCompletionsPayload = { model: 'test-model', messages }; - const invocation: ChatCompletionsInvocation = { - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), - }; - await withRoleCompatibilityApplied(invocation, gatewayCtx, okEvents); - return invocation.payload.messages; -}; - -test('leaves roles unchanged without flags or at a translated target', async () => { - const messages: ChatCompletionsMessage[] = [ - { role: 'system', content: 'rules' }, - { role: 'developer', content: 'developer rules' }, - ]; - - assertEquals(await applyRoles(messages, new Set()), messages); - assertEquals( - await applyRoles(messages, new Set(['rewrite-system-to-developer']), 'responses'), - messages, - ); -}); - -test('applies the system and developer rewrites independently', async () => { - assertEquals( - await applyRoles( - [{ role: 'system', content: 'rules' }, { role: 'user', content: 'hello' }], - new Set(['rewrite-system-to-developer']), - ), - [{ role: 'developer', content: 'rules' }, { role: 'user', content: 'hello' }], - ); - assertEquals( - await applyRoles( - [{ role: 'developer', content: 'rules' }, { role: 'user', content: 'hello' }], - new Set(['rewrite-developer-to-system']), - ), - [{ role: 'system', content: 'rules' }, { role: 'user', content: 'hello' }], - ); -}); - -test('preserves the leading system run and rewrites later system messages', async () => { - assertEquals( - await applyRoles( - [ - { role: 'system', content: 'base A' }, - { role: 'system', content: 'base B' }, - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline rules' }, - ], - new Set(['rewrite-mid-conv-system-to-user']), - ), - [ - { role: 'system', content: 'base A' }, - { role: 'system', content: 'base B' }, - { role: 'user', content: 'hello' }, - { role: 'user', content: 'inline rules' }, - ], - ); -}); - -test('keeps a leading-only system run and an empty input unchanged', async () => { - const leading: ChatCompletionsMessage[] = [ - { role: 'system', content: 'base A' }, - { role: 'system', content: 'base B' }, - ]; - assertEquals(await applyRoles(leading, new Set(['rewrite-mid-conv-system-to-user'])), leading); - assertEquals(await applyRoles([], new Set(['rewrite-mid-conv-system-to-user'])), []); -}); - -test('preserves multipart content identity when rewriting a mid-conversation system message', async () => { - const content = [ - { type: 'text' as const, text: 'one' }, - { type: 'text' as const, text: 'two' }, - ]; - const result = await applyRoles( - [{ role: 'user', content: 'hello' }, { role: 'system', content }], - new Set(['rewrite-mid-conv-system-to-user']), - ); - assertEquals(result, [{ role: 'user', content: 'hello' }, { role: 'user', content }]); - assert(result[1]?.content === content); -}); - -test('applies overlapping flags in system-to-developer then developer-to-system order', async () => { - assertEquals( - await applyRoles( - [ - { role: 'system', content: 'base rules' }, - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline rules' }, - ], - new Set([ - 'rewrite-system-to-developer', - 'rewrite-developer-to-system', - 'rewrite-mid-conv-system-to-user', - ]), - ), - [ - { role: 'system', content: 'base rules' }, - { role: 'user', content: 'hello' }, - { role: 'user', content: 'inline rules' }, - ], - ); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice_test.ts deleted file mode 100644 index 32e5a2de36..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice_test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { test } from 'vitest'; - -import { withReasoningDisabledOnForcedToolChoice } from '../../../../../src/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts'; -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const invocation = ( - payload: ChatCompletionsPayload, - enabledFlags: ReadonlySet = new Set(['disable-reasoning-on-forced-tool-choice']), -): ChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'chat-completions', - headers: new Headers(), -}); - -test('required tool_choice sets reasoning_effort to the canonical none sentinel', async () => { - const input = invocation({ - model: 'm', - messages: [], - reasoning_effort: 'high', - tool_choice: 'required', - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning_effort, 'none'); - const out = input.payload as unknown as Record; - assertEquals(out.thinking, undefined); - assertEquals(out.enable_thinking, undefined); -}); - -test('object tool_choice is treated as forced', async () => { - const input = invocation({ - model: 'm', - messages: [], - reasoning_effort: 'high', - tool_choice: { type: 'function', function: { name: 'x' } }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning_effort, 'none'); -}); - -test('non-forced tool_choice leaves reasoning_effort untouched', async () => { - for (const tool_choice of ['auto', 'none', null] as const) { - const input = invocation({ - model: 'm', - messages: [], - reasoning_effort: 'high', - tool_choice, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning_effort, 'high'); - } -}); - -test('is a no-op when the flag is not set on the candidate', async () => { - const input = invocation( - { model: 'm', messages: [], reasoning_effort: 'high', tool_choice: 'required' }, - new Set(), - ); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning_effort, 'high'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/include-usage-stream-options_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/include-usage-stream-options_test.ts deleted file mode 100644 index aa1eeb39f0..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/include-usage-stream-options_test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { test } from 'vitest'; - -import { withUsageStreamOptionsIncluded } from '../../../../../src/data-plane/chat/chat-completions/interceptors/include-usage-stream-options.ts'; -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const invocation = (payload: ChatCompletionsPayload): ChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'chat-completions', - headers: new Headers(), -}); - -test('adds stream_options.include_usage when stream_options is absent', async () => { - const input = invocation({ model: 'm', messages: [] }); - - await withUsageStreamOptionsIncluded(input, stubCtx, okEvents); - - assertEquals(input.payload.stream_options, { include_usage: true }); -}); - -test('overrides include_usage:false on an existing stream_options object', async () => { - const input = invocation({ - model: 'm', - messages: [], - stream_options: { include_usage: false }, - }); - - await withUsageStreamOptionsIncluded(input, stubCtx, okEvents); - - assertEquals(input.payload.stream_options, { include_usage: true }); -}); - -test('preserves sibling stream_options keys while forcing include_usage on', async () => { - // Upstreams that pass through unknown fields can carry extra `stream_options` - // keys past our typed surface; the interceptor must not drop them when it - // flips include_usage. - const input = invocation({ - model: 'm', - messages: [], - stream_options: { extra: 'keep-me', include_usage: false } as unknown as ChatCompletionsPayload['stream_options'], - }); - - await withUsageStreamOptionsIncluded(input, stubCtx, okEvents); - - assertEquals(input.payload.stream_options as unknown as Record, { - extra: 'keep-me', - include_usage: true, - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens_test.ts deleted file mode 100644 index 6640ddc349..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens_test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { expect, test } from 'vitest'; - -import { withExclusiveCachedTokensNormalized } from '../../../../../src/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens.ts'; -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { billableUsageFromChatCompletionsUsage } from '../../../../../src/data-plane/chat/chat-completions/usage.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = ( - enabledFlags: ReadonlySet = new Set(['usage-exclusive-cached-tokens']), - targetApi: ChatCompletionsInvocation['targetApi'] = 'chat-completions', -): ChatCompletionsInvocation => ({ - payload: { model: 'kimi-k3', messages: [{ role: 'user', content: 'hi' }] } satisfies ChatCompletionsPayload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), -}); - -const collectFrames = async (result: ExecuteResult>): Promise[]> => { - if (result.type !== 'events') throw new Error('expected events result'); - const out: ProtocolFrame[] = []; - for await (const frame of result.events) out.push(frame); - return out; -}; - -const usageChunk = (usage: Record): ChatCompletionsStreamEvent => ({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'kimi-k3', - choices: [], - usage: usage as unknown as ChatCompletionsStreamEvent['usage'], -}); - -const run = async (ctx: ChatCompletionsInvocation, usage: Record): Promise> => { - const result = await withExclusiveCachedTokensNormalized(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame(usageChunk(usage)); - })(), - testTelemetryModelIdentity, - ))); - const frames = await collectFrames(result); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - return frame.event.usage as unknown as Record; -}; - -// Verbatim from a Charm Hyper kimi-k3 turn: 479 + 13312 + 373 = 14164, so -// `prompt_tokens` counts only what the cache did not serve, and total_tokens -// witnesses it. -const HYPER_USAGE = { - prompt_tokens: 479, - completion_tokens: 373, - total_tokens: 14164, - prompt_tokens_details: { cached_tokens: 13312 }, -}; - -// The same accounting with the total withheld, which is the only state the -// flag exists to settle. -const HYPER_USAGE_WITHOUT_TOTAL = { ...HYPER_USAGE, total_tokens: undefined }; - -// An ordinary OpenAI-shaped turn: 1000 + 50 = 1050, cached inside the input. -const INCLUSIVE_USAGE = { - prompt_tokens: 1000, - completion_tokens: 50, - total_tokens: 1050, - prompt_tokens_details: { cached_tokens: 400 }, -}; - -test('folds on the totals alone, with no flag set', async () => { - const usage = await run(invocation(new Set()), HYPER_USAGE); - assertEquals(usage.prompt_tokens, 13791); - // total_tokens already counted the real input, and now agrees with it. - assertEquals(usage.total_tokens, 14164); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 13312 }); -}); - -test('folds on the flag when the totals witness nothing', async () => { - const usage = await run(invocation(), HYPER_USAGE_WITHOUT_TOTAL); - assertEquals(usage.prompt_tokens, 13791); -}); - -test('leaves a normalized chunk billable instead of underflowing', async () => { - const raw = HYPER_USAGE as unknown as NonNullable; - expect(() => billableUsageFromChatCompletionsUsage(raw, null)).toThrowError(RangeError); - - const usage = await run(invocation(new Set()), HYPER_USAGE); - assertEquals( - billableUsageFromChatCompletionsUsage(usage as unknown as NonNullable, null), - { input: 479, cacheRead: 13312, cacheWrite: 0, cacheWrite1h: 0, output: 373 }, - ); -}); - -test('folds cache writes back as well', async () => { - const usage = await run(invocation(), { - prompt_tokens: 100, - completion_tokens: 10, - prompt_tokens_details: { cached_tokens: 120, cache_creation_input_tokens: 80 }, - }); - assertEquals(usage.prompt_tokens, 300); -}); - -test('leaves an inclusive chunk alone when no flag claims otherwise', async () => { - const usage = await run(invocation(new Set()), INCLUSIVE_USAGE); - assertEquals(usage.prompt_tokens, 1000); -}); - -test('leaves a chunk with no cache counts untouched', async () => { - const usage = await run(invocation(), { prompt_tokens: 40, completion_tokens: 1, total_tokens: 41 }); - assertEquals(usage.prompt_tokens, 40); -}); - -test('raises when the flag claims exclusive and the totals say inclusive', async () => { - await expect(run(invocation(), INCLUSIVE_USAGE)).rejects.toThrowError(/usage-exclusive-cached-tokens is enabled/); -}); - -test('raises naming the flag when the cache counts underflow with no verdict', async () => { - await expect(run(invocation(new Set()), HYPER_USAGE_WITHOUT_TOTAL)) - .rejects.toThrowError(/enable usage-exclusive-cached-tokens/); -}); - -test('stands down entirely when the wire it speaks about is elsewhere', async () => { - // A Chat Completions request whose upstream speaks Responses: the Responses - // entry owns that wire, and by the time these events reach this chain they - // are a translation. Neither the fold nor its errors apply here, so even a - // payload that would raise on the wire passes through untouched. - const usage = await run(invocation(undefined, 'responses'), HYPER_USAGE_WITHOUT_TOTAL); - assertEquals(usage.prompt_tokens, 479); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/normalize-usage_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/normalize-usage_test.ts deleted file mode 100644 index 6c7ebe2038..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/normalize-usage_test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { test } from 'vitest'; - -import { withUsageNormalized } from '../../../../../src/data-plane/chat/chat-completions/interceptors/normalize-usage.ts'; -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: ChatCompletionsPayload = { model: 'test-model', messages: [] }): ChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'chat-completions', - headers: new Headers(), -}); - -const collectFrames = async (result: ExecuteResult>): Promise[]> => { - if (result.type !== 'events') throw new Error('expected events result'); - const out: ProtocolFrame[] = []; - for await (const frame of result.events) out.push(frame); - return out; -}; - -const runWithFrames = async (...frames: ProtocolFrame[]): Promise[]> => { - const result = await withUsageNormalized(invocation(), stubCtx, () => - Promise.resolve( - eventResult( - (async function* () { - for (const frame of frames) yield frame; - })(), - testTelemetryModelIdentity, - ), - )); - return await collectFrames(result); -}; - -const usageRecord = (usage: NonNullable): Record => usage as unknown as Record; - -test('leaves a spec-compliant carrier usage chunk untouched', async () => { - const frames = await runWithFrames( - eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'gpt-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_tokens_details: { cached_tokens: 60, audio_tokens: 0 }, - } as unknown as ChatCompletionsStreamEvent['usage'], - }), - ); - - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 60, audio_tokens: 0 }); -}); - -test('relocates usage from a non-empty choices chunk onto a synthesized carrier', async () => { - const frames = await runWithFrames( - eventFrame({ - id: 'chatcmpl_1', - object: 'chat.completion.chunk', - created: 1, - model: 'gpt-test', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_tokens_details: { cached_tokens: 70 }, - } as unknown as ChatCompletionsStreamEvent['usage'], - }), - ); - - assertEquals(frames.length, 2); - - const first = frames[0]; - if (first.type !== 'event') throw new Error('expected event frame'); - assertEquals(first.event.choices, [{ index: 0, delta: {}, finish_reason: 'stop' }]); - assertEquals(first.event.usage, undefined); - - const carrier = frames[1]; - if (carrier.type !== 'event') throw new Error('expected event frame'); - assertEquals(carrier.event.id, 'chatcmpl_1'); - assertEquals(carrier.event.model, 'gpt-test'); - assertEquals(carrier.event.choices, []); - const usage = usageRecord(carrier.event.usage!); - assertEquals(usage.prompt_tokens, 100); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 70 }); -}); - -test('passes raw vendor cache fields through unchanged so vendor interceptors stay responsible for the rewrite', async () => { - const frames = await runWithFrames( - eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'unknown-vendor', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_cache_hit_tokens: 70, - cached_tokens: 25, - } as unknown as ChatCompletionsStreamEvent['usage'], - }), - ); - - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_cache_hit_tokens, 70); - assertEquals(usage.cached_tokens, 25); - assertEquals('prompt_tokens_details' in usage, false); -}); - -test('leaves chunks without usage untouched', async () => { - const original = eventFrame({ - id: 'chatcmpl_3', - object: 'chat.completion.chunk', - created: 1, - model: 'gpt-test', - choices: [ - { index: 0, delta: { content: 'hi' }, finish_reason: null }, - ], - } satisfies ChatCompletionsStreamEvent); - - const frames = await runWithFrames(original); - - assertEquals(frames, [original]); -}); - -test('passes protocol done frames through verbatim', async () => { - const done = doneFrame(); - - const frames = await runWithFrames(done); - - assertEquals(frames, [done]); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key_test.ts deleted file mode 100644 index 5f8b6b7cbc..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key_test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { test } from 'vitest'; - -import { withPromptCacheKeyStripped } from '../../../../../src/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key.ts'; -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const invocation = ( - payload: ChatCompletionsPayload, - enabledFlags: ReadonlySet = new Set(['strip-prompt-cache-key']), -): ChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'chat-completions', - headers: new Headers(), -}); - -test('drops top-level prompt_cache_key when the flag is on', async () => { - const input = invocation({ - model: 'm', - messages: [], - prompt_cache_key: 'thread-42', - }); - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(Object.hasOwn(input.payload, 'prompt_cache_key'), false); -}); - -test('leaves the payload untouched when prompt_cache_key is absent', async () => { - const input = invocation({ model: 'm', messages: [] }); - const before = input.payload; - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(input.payload, before); -}); - -test('is a no-op when the flag is not set on the candidate', async () => { - const input = invocation( - { model: 'm', messages: [], prompt_cache_key: 'thread-42' }, - new Set(), - ); - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(input.payload.prompt_cache_key, 'thread-42'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize_test.ts deleted file mode 100644 index 0a5e6c2f5b..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize_test.ts +++ /dev/null @@ -1,393 +0,0 @@ -import { expect, test } from 'vitest'; - -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { withVendorDeepSeekChatCompletionsNormalize } from '../../../../../src/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -type DeepSeekReasoningDelta = ChatCompletionsStreamEvent['choices'][number]['delta'] & { - reasoning_content?: string; -}; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: ChatCompletionsPayload, enabledFlags: ReadonlySet = new Set(['vendor-deepseek'])): ChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'chat-completions', - headers: new Headers(), -}); - -const baseRequest = (): ChatCompletionsPayload => ({ - model: 'deepseek-reasoner', - messages: [ - { role: 'user', content: 'first turn' }, - { - role: 'assistant', - content: null, - reasoning_text: 'let me check the docs', - reasoning_opaque: 'opaque-blob', - reasoning_items: [{ type: 'reasoning', summary: [] }], - tool_calls: [ - { id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{}' } }, - ], - }, - { role: 'tool', tool_call_id: 'call_1', content: 'result' }, - { role: 'user', content: 'next turn' }, - ], -}); - -const collectFrames = async (result: ExecuteResult>): Promise[]> => { - if (result.type !== 'events') throw new Error('expected events result'); - const out: ProtocolFrame[] = []; - for await (const frame of result.events) out.push(frame); - return out; -}; - -const usageRecord = (usage: NonNullable): Record => usage as unknown as Record; - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -// ── Outbound: assistant reasoning field rewrite ── - -test('renames outbound reasoning_text to reasoning_content on assistant messages', async () => { - const ctx = invocation(baseRequest()); - - let observed: ChatCompletionsPayload | null = null; - await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const assistant = observed!.messages[1] as unknown as Record; - assertEquals(assistant.reasoning_content, 'let me check the docs'); - assertEquals(assistant.reasoning_text, undefined); - assertEquals(assistant.reasoning_opaque, undefined); - assertEquals(assistant.reasoning_items, undefined); - assertEquals((assistant.tool_calls as unknown[]).length, 1); -}); - -test('synthesizes reasoning_content from reasoning_items when reasoning_text is absent', async () => { - const ctx = invocation({ - model: 'deepseek-reasoner', - messages: [ - { role: 'user', content: 'first turn' }, - { - role: 'assistant', - content: null, - reasoning_items: [ - { - type: 'reasoning', - id: 'rs_1', - summary: [ - { type: 'summary_text', text: 'step one. ' }, - { type: 'summary_text', text: 'step two.' }, - ], - }, - ], - tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{}' } }], - }, - { role: 'tool', tool_call_id: 'call_1', content: 'result' }, - ], - }); - - let observed: ChatCompletionsPayload | null = null; - await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const assistant = observed!.messages[1] as unknown as Record; - assertEquals(assistant.reasoning_content, 'step one. step two.'); - assertEquals(assistant.reasoning_text, undefined); - assertEquals(assistant.reasoning_opaque, undefined); - assertEquals(assistant.reasoning_items, undefined); -}); - -test('strips reasoning_items even when no summaries are available', async () => { - const ctx = invocation({ - model: 'deepseek-reasoner', - messages: [ - { role: 'user', content: 'first turn' }, - { - role: 'assistant', - content: 'answer', - reasoning_items: [{ type: 'reasoning' }], - reasoning_opaque: 'opaque-chain', - }, - ], - }); - - let observed: ChatCompletionsPayload | null = null; - await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const assistant = observed!.messages[1] as unknown as Record; - assertEquals(assistant.reasoning_content, undefined); - assertEquals(assistant.reasoning_items, undefined); - assertEquals(assistant.reasoning_opaque, undefined); - assertEquals(assistant.content, 'answer'); -}); - -// ── Outbound: reasoning_effort === 'none' canonical sentinel ── - -test("translates canonical reasoning_effort: 'none' into top-level thinking:{type:'disabled'}", async () => { - const ctx = invocation({ - model: 'deepseek-reasoner', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'none', - }); - - let observed: ChatCompletionsPayload | null = null; - await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const out = observed! as unknown as Record; - assertEquals(out.reasoning_effort, undefined); - assertEquals(out.thinking, { type: 'disabled' }); -}); - -test('leaves a real reasoning_effort value untouched (only the none sentinel triggers the rewrite)', async () => { - const ctx = invocation({ - model: 'deepseek-reasoner', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'high', - }); - - let observed: ChatCompletionsPayload | null = null; - await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.reasoning_effort, 'high'); - const out = observed! as unknown as Record; - assertEquals(out.thinking, undefined); -}); - -// ── Outbound: structured-output json_schema downgrade ── - -test('downgrades response_format json_schema to json_object (schema body dropped)', async () => { - const ctx = invocation({ - model: 'deepseek-chat', - messages: [{ role: 'user', content: 'hi' }], - response_format: { - type: 'json_schema', - json_schema: { - name: 'messages_response', - strict: true, - schema: { type: 'object', properties: { test: { type: 'string' } }, required: ['test'], additionalProperties: false }, - }, - }, - }); - - let observed: ChatCompletionsPayload | null = null; - await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.response_format, { type: 'json_object' }); -}); - -test('leaves an already-json_object response_format untouched', async () => { - const ctx = invocation({ - model: 'deepseek-chat', - messages: [{ role: 'user', content: 'hi' }], - response_format: { type: 'json_object' }, - }); - - let observed: ChatCompletionsPayload | null = null; - await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.response_format, { type: 'json_object' }); -}); - -// ── Inbound: delta reasoning_content → reasoning_text ── - -test('renames inbound protocol reasoning_content deltas to reasoning_text', async () => { - const ctx = invocation(baseRequest()); - const upstreamChunk: ChatCompletionsStreamEvent = { - id: 'chunk_1', - object: 'chat.completion.chunk', - created: 1, - model: 'deepseek-reasoner', - choices: [ - { - index: 0, - delta: { reasoning_content: 'thinking...' } as DeepSeekReasoningDelta, - finish_reason: null, - }, - ], - }; - - const result = await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { yield eventFrame(upstreamChunk); })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const delta = frame.event.choices[0].delta as Record; - assertEquals(delta.reasoning_text, 'thinking...'); - assertEquals(delta.reasoning_content, undefined); -}); - -test('preserves reasoning_content from non-stream JSON responses', async () => { - const ctx = invocation(baseRequest()); - const id = 'chatcmpl_deepseek_json'; - const model = 'deepseek-reasoner'; - const chunk = (delta: ChatCompletionsStreamEvent['choices'][number]['delta'], finish_reason: 'stop' | null = null): ChatCompletionsStreamEvent => ({ - id, - object: 'chat.completion.chunk', - created: 1, - model, - choices: [{ index: 0, delta, finish_reason }], - }); - - const result = await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame(chunk({ role: 'assistant' })); - yield eventFrame(chunk({ reasoning_content: 'json thinking' } as ChatCompletionsStreamEvent['choices'][number]['delta'])); - yield eventFrame(chunk({ content: 'answer' })); - yield eventFrame(chunk({}, 'stop')); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - const reasoningFrame = frames.find(frame => frame.type === 'event' && frame.event.choices[0]?.delta.reasoning_text !== undefined); - assertEquals(reasoningFrame?.type === 'event' ? reasoningFrame.event.choices[0]?.delta.reasoning_text : undefined, 'json thinking'); -}); - -// ── Inbound: usage cache-token field rewrite ── - -test('rewrites prompt_cache_hit_tokens/prompt_cache_miss_tokens into prompt_tokens_details.cached_tokens', async () => { - const ctx = invocation(baseRequest()); - const result = await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'deepseek-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_cache_hit_tokens: 70, - prompt_cache_miss_tokens: 30, - prompt_tokens_details: { upstream_metric: 'preserved' }, - } as unknown as ChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens, 100); - assertEquals(usage.prompt_tokens_details, { upstream_metric: 'preserved', cached_tokens: 70 }); - assertEquals('prompt_cache_hit_tokens' in usage, false); - assertEquals('prompt_cache_miss_tokens' in usage, false); -}); - -test('replaces array-shaped prompt_tokens_details without copying array indices', async () => { - const ctx = invocation(baseRequest()); - const result = await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'deepseek-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_cache_hit_tokens: 70, - prompt_cache_miss_tokens: 30, - prompt_tokens_details: [{ cached_tokens: 1 }], - } as unknown as ChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 70 }); -}); - -test('propagates upstream stream errors unchanged', async () => { - const failure = new Error('upstream stream failed'); - const result = await withVendorDeepSeekChatCompletionsNormalize(invocation(baseRequest()), stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - throw failure; - })(), - testTelemetryModelIdentity, - ))); - - await expect(collectFrames(result)).rejects.toBe(failure); -}); - -// ── Pass-through ── - -test('leaves protocol done frames untouched', async () => { - const ctx = invocation(baseRequest()); - const done = doneFrame(); - - const result = await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { yield done; })(), - testTelemetryModelIdentity, - ))); - - assertEquals(await collectFrames(result), [done]); -}); - -test('early-returns when its flag is not set on the candidate', async () => { - const ctx = invocation( - { - model: 'deepseek-reasoner', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'none', - }, - new Set(), - ); - - let observed: ChatCompletionsPayload | null = null; - await withVendorDeepSeekChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.reasoning_effort, 'none'); - const out = observed! as unknown as Record; - assertEquals(out.thinking, undefined); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize_test.ts deleted file mode 100644 index 235d9a59c3..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize_test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { expect, test } from 'vitest'; - -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { withVendorKimiChatCompletionsNormalize } from '../../../../../src/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: ChatCompletionsPayload, enabledFlags: ReadonlySet = new Set(['vendor-kimi'])): ChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'chat-completions', - headers: new Headers(), -}); - -const collectFrames = async (result: ExecuteResult>): Promise[]> => { - if (result.type !== 'events') throw new Error('expected events result'); - const out: ProtocolFrame[] = []; - for await (const frame of result.events) out.push(frame); - return out; -}; - -const usageRecord = (usage: NonNullable): Record => usage as unknown as Record; - -const baseRequest = (): ChatCompletionsPayload => ({ model: 'kimi-k2', messages: [{ role: 'user', content: 'hi' }] }); - -test('rewrites flat cached_tokens into prompt_tokens_details.cached_tokens', async () => { - const ctx = invocation(baseRequest()); - const result = await withVendorKimiChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'kimi-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - cached_tokens: 50, - prompt_tokens_details: { upstream_metric: 'preserved' }, - } as unknown as ChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens_details, { upstream_metric: 'preserved', cached_tokens: 50 }); - assertEquals('cached_tokens' in usage, false); -}); - -test('replaces array-shaped prompt_tokens_details without copying array indices', async () => { - const ctx = invocation(baseRequest()); - const result = await withVendorKimiChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'kimi-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - cached_tokens: 50, - prompt_tokens_details: [{ cached_tokens: 1 }], - } as unknown as ChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 50 }); -}); - -test('propagates upstream stream errors unchanged', async () => { - const failure = new Error('upstream stream failed'); - const result = await withVendorKimiChatCompletionsNormalize(invocation(baseRequest()), stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - throw failure; - })(), - testTelemetryModelIdentity, - ))); - - await expect(collectFrames(result)).rejects.toBe(failure); -}); - -test('early-returns when its flag is not set on the candidate', async () => { - const ctx = invocation(baseRequest(), new Set()); - const result = await withVendorKimiChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'kimi-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - cached_tokens: 50, - } as unknown as ChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.cached_tokens, 50); - assertEquals('prompt_tokens_details' in usage, false); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize_test.ts deleted file mode 100644 index abcb9de7a7..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize_test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { test } from 'vitest'; - -import type { ChatCompletionsInvocation } from '../../../../../src/data-plane/chat/chat-completions/interceptors/types.ts'; -import { withVendorQwenChatCompletionsNormalize } from '../../../../../src/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: ChatCompletionsPayload, enabledFlags: ReadonlySet = new Set(['vendor-qwen'])): ChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'chat-completions', - headers: new Headers(), -}); - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -test("translates canonical reasoning_effort: 'none' into top-level enable_thinking:false", async () => { - const ctx = invocation({ - model: 'qwen-max', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'none', - }); - - let observed: ChatCompletionsPayload | null = null; - await withVendorQwenChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const out = observed! as unknown as Record; - assertEquals(out.reasoning_effort, undefined); - assertEquals(out.enable_thinking, false); -}); - -test('leaves a real reasoning_effort value untouched (only the none sentinel triggers the rewrite)', async () => { - const ctx = invocation({ - model: 'qwen-max', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'high', - }); - - let observed: ChatCompletionsPayload | null = null; - await withVendorQwenChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.reasoning_effort, 'high'); - const out = observed! as unknown as Record; - assertEquals(out.enable_thinking, undefined); -}); - -test('early-returns when its flag is not set on the candidate', async () => { - const ctx = invocation( - { - model: 'qwen-max', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'none', - }, - new Set(), - ); - - let observed: ChatCompletionsPayload | null = null; - await withVendorQwenChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.reasoning_effort, 'none'); - const out = observed! as unknown as Record; - assertEquals(out.enable_thinking, undefined); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/respond_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/respond_test.ts deleted file mode 100644 index e6aa47fbd4..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/respond_test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { Hono } from 'hono'; -import { expect, test } from 'vitest'; - -import { respondChatCompletions } from '../../../../src/data-plane/chat/chat-completions/respond.ts'; -import type { DumpAccumulator } from '../../../../src/dump/accumulator.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventResult } from '@floway-dev/provider'; -import { assert, assertEquals, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const recordingDump = () => { - const frames: ProtocolFrame[] = []; - return { - frames, - dump: { - failed: () => {}, - success: () => {}, - frame: (frame: ProtocolFrame) => { frames.push(frame); }, - } as unknown as DumpAccumulator, - }; -}; - -const chunk = (text: string): ChatCompletionsStreamEvent => ({ - id: 'x', object: 'chat.completion.chunk', created: 0, model: 'm', - choices: [{ index: 0, delta: { content: text }, finish_reason: null }], -}); - -const serve = async (dump: DumpAccumulator, frames: AsyncGenerator>): Promise => { - const ctx = mockChatGatewayCtx({ wantsStream: true, dump }); - const app = new Hono().get('/', c => - respondChatCompletions(c, eventResult(frames, testTelemetryModelIdentity), true, true, ctx)); - return await (await app.request('/')).text(); -}; - -test('the error frame the client is sent is recorded like every other frame', async () => { - const { dump, frames } = recordingDump(); - const body = await serve(dump, (async function* () { - yield eventFrame(chunk('hi')); - throw new RangeError('cache token counts exceed inclusive input tokens: 479 - 13312 - 0'); - })()); - - // What the client receives is unchanged by this. - expect(body).toContain('event: error'); - expect(body).toContain('cache token counts exceed inclusive input tokens'); - expect(body).not.toContain('data: [DONE]'); - - // And the recorded turn now ends on the same event rather than on the last - // good chunk. - assertEquals(frames.length, 2); - const last = frames[1]; - assert(last.type === 'event', 'expected an event frame'); - const payload = last.event as unknown as { error?: { message?: string } }; - assertEquals(payload.error?.message, 'cache token counts exceed inclusive input tokens: 479 - 13312 - 0'); -}); - -test('a stream that completes records only its own frames', async () => { - const { dump, frames } = recordingDump(); - const body = await serve(dump, (async function* () { - yield eventFrame(chunk('hi')); - yield doneFrame(); - })()); - - expect(body).toContain('data: [DONE]'); - assertEquals(frames.length, 2); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/chat-completions/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/chat-completions/serve_test.ts deleted file mode 100644 index 24584af00b..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/chat-completions/serve_test.ts +++ /dev/null @@ -1,398 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; - -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ModelCandidate, directFetcher, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; - -// Mock the resolver seam so each test hands the serve exactly the model -// candidates it wants, optionally with an alias-rules overlay attached. -interface QueuedResolution { - readonly candidates: readonly ModelCandidate[]; - readonly sawModel: boolean; - readonly failedUpstreams: readonly string[]; -} -const resolutionsQueue: QueuedResolution[] = []; -const lastResolveCall: { model?: string } = {}; -vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { - const original = await importOriginal(); - return { - ...original, - enumerateModelCandidates: vi.fn(async ({ model }: { model: string }) => { - lastResolveCall.model = model; - const next = resolutionsQueue.shift(); - if (next === undefined) throw new Error('serve_test: no resolution enqueued'); - return next; - }), - }; -}); - -const { chatCompletionsServe } = await import('../../../../src/data-plane/chat/chat-completions/serve.ts'); - -const API_KEY_ID = 'key_chat_completions_serve_test'; - -const queueResolution = ( - candidates: readonly ModelCandidate[], - extra: { sawModel?: boolean; aliasRules?: AliasRules } = {}, -): void => { - const rules = extra.aliasRules; - resolutionsQueue.push({ - candidates: rules !== undefined ? candidates.map(c => ({ ...c, rules })) : candidates, - sawModel: extra.sawModel ?? candidates.length > 0, - failedUpstreams: [], - }); -}; - -afterEach(() => { resolutionsQueue.length = 0; }); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): ChatCompletionsPayload => ({ - model: 'test-model', - messages: [{ role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeChatCompletionsEvents = (): readonly ChatCompletionsStreamEvent[] => [ - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }], - }, - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }, - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, -]; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - modelId?: string; - endpoints?: ModelEndpoints; - callChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ - callChatCompletions: overrides.callChatCompletions, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, - }, - model: stubInternalModel({ - id: overrides.modelId ?? 'test-model', - ...(overrides.endpoints ? { endpoints: overrides.endpoints } : {}), - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -test('generate routes a native Chat Completions candidate end to end', async () => { - installRepo(); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'test-model-key', headers: new Headers(), - })); - queueResolution([makeCandidate({ upstream: 'up_a', callChatCompletions })]); - - const result = await chatCompletionsServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - const events = await collectEvents(result.events); - assert(events.length >= 1); - assertEquals(callChatCompletions.mock.calls.length, 1); -}); - -test('generate filters out candidates that do not expose any chat-completions-target endpoint', async () => { - installRepo(); - const callChatCompletions = vi.fn(); - // `completions:{}` is not in the chatCompletionsTarget preference list - // (`chat-completions` > `messages` > `responses`), so the picker rejects - // this candidate. - queueResolution([makeCandidate({ upstream: 'up_m', endpoints: { completions: {} }, callChatCompletions })]); - - const result = await chatCompletionsServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - // canServe drops messages-only candidates; with no viable candidate the - // serve renders model-unsupported as a 400 api-error (distinct from the - // model-missing 404) without ever reaching the upstream call. - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 400); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callChatCompletions.mock.calls.length, 0); -}); - -test('generate falls through to the next candidate when the first yields an upstream error', async () => { - installRepo(); - const originalImageUrl = 'data:image/png;base64,AQID'; - const payload = makePayload({ - messages: [{ - role: 'user', - content: [{ type: 'image_url', image_url: { url: originalImageUrl, detail: 'auto' } }], - }], - }); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (_model: unknown, body: unknown): Promise> => { - const message = (body as Omit).messages[0]; - if (!Array.isArray(message.content) || message.content[0]?.type !== 'image_url') throw new Error('expected image content'); - message.content[0].image_url.url = 'data:image/webp;base64,COMPRESSED'; - return { ok: false, response: firstError, modelKey: 'first-key' }; - }); - let fallbackImageUrl: string | undefined; - const secondCall = vi.fn(async (_model: unknown, body: unknown): Promise> => { - const message = (body as Omit).messages[0]; - if (!Array.isArray(message.content) || message.content[0]?.type !== 'image_url') throw new Error('expected image content'); - fallbackImageUrl = message.content[0].image_url.url; - return { ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'second-key', headers: new Headers() }; - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callChatCompletions: firstCall }), - makeCandidate({ upstream: 'up_b', callChatCompletions: secondCall }), - ]); - - const result = await chatCompletionsServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - // The narrowed candidate list exists exactly so a transient upstream - // failure (5xx/429/network) on one entry rolls over to the next. The - // second candidate's success is the request's final answer. - assertEquals(result.type, 'events'); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(fallbackImageUrl, originalImageUrl); - const sourceMessage = payload.messages[0]; - if (!Array.isArray(sourceMessage.content) || sourceMessage.content[0]?.type !== 'image_url') throw new Error('expected source image content'); - assertEquals(sourceMessage.content[0].image_url.url, originalImageUrl); -}); - -// A mid-attempt throw (interceptor bug / translation error / provider-layer -// JS exception not represented as a ChatServeFailure) must attribute the perf -// error row to the throwing candidate, not the previous one that already -// failed cleanly with a 5xx. -test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => { - throw new Error('simulated provider-layer JS exception'); - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callChatCompletions: firstCall }), - makeCandidate({ upstream: 'up_b', callChatCompletions: secondCall }), - ]); - - const ctx = makeGatewayCtx(); - await chatCompletionsServe.generate({ - payload: makePayload(), - ctx, - headers: new Headers(), - }).then( - () => { throw new Error('expected chatCompletionsServe.generate to throw'); }, - (error: unknown) => { - assertEquals((error as Error).message, 'simulated provider-layer JS exception'); - }, - ); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(ctx.attempt.telemetry?.upstream, 'up_b'); -}); - -test('generate surfaces the last upstream error verbatim when every candidate fails', async () => { - installRepo(); - const firstError = new Response('first', { status: 503 }); - const lastError = new Response(JSON.stringify({ error: { message: 'last' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callChatCompletions: async () => ({ ok: false, response: firstError, modelKey: 'first-key' }) }), - makeCandidate({ upstream: 'up_b', callChatCompletions: async () => ({ ok: false, response: lastError, modelKey: 'last-key' }) }), - ]); - - const result = await chatCompletionsServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 502); -}); - -test('generate is a routing no-op when the payload carries no reasoning carriers (degenerate path)', async () => { - installRepo(); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'test-model-key', headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', callChatCompletions }), - makeCandidate({ upstream: 'up_b', callChatCompletions }), - ]); - - const result = await chatCompletionsServe.generate({ - // A bare user message: no reasoning blocks → affinity walk finds no - // refs → both candidates surface in the original order. - payload: makePayload({ messages: [{ role: 'user', content: 'hi' }] }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callChatCompletions.mock.calls.length, 1); -}); - -test('generate renders model-missing when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await chatCompletionsServe.generate({ - payload: makePayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assertEquals(body.error.message, 'Model unknown-model is not available on any configured upstream.'); -}); - -test('alias resolution swaps the inbound model id for the target and overlays rules onto the IR', async () => { - installRepo(); - const capturedBodies: ChatCompletionsPayload[] = []; - const observedModelIds: string[] = []; - const callChatCompletions = vi.fn(async (model: unknown, body: unknown): Promise> => { - observedModelIds.push((model as { id: string }).id); - capturedBodies.push(body as ChatCompletionsPayload); - return { ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'gpt-5.4', headers: new Headers() }; - }); - // Alias flow shape: the resolver returns candidates carrying the target's - // upstream catalog id AND the alias's rule overlay on `candidate.rules`. - // The attempt stamps its private clone with `candidate.model.id` and reads - // the overlay directly off `candidate.rules` at wire-call time. - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'gpt-5.4', callChatCompletions }); - queueResolution([candidate], { aliasRules: { reasoning: { effort: 'low' }, verbosity: 'low' } }); - - const payload = makePayload({ model: 'gpt-fast' }); - const result = await chatCompletionsServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - - // The resolver saw the inbound alias id verbatim — target-id walking - // happens inside the resolver, not in serve. - assertEquals(lastResolveCall.model, 'gpt-fast'); - assertEquals(observedModelIds, ['gpt-5.4']); - assertEquals(payload.model, 'gpt-fast'); - // Alias rules land on the IR through candidate.rules → the attempt's - // applyRulesToUpstreamChatCompletions call. - const observed = capturedBodies[0]!; - assertEquals(observed.reasoning_effort, 'low'); - assertEquals(observed.verbosity, 'low'); -}); - -test('direct dispatch uses the resolved public id without mutating the addressed model', async () => { - // A prefix-addressable id ('cop/gpt-5.4') resolves to the catalog's - // 'gpt-5.4' — the resolver strips the prefix internally. The attempt pins - // only its private clone to that canonical id. - installRepo(); - const observedModelIds: string[] = []; - const callChatCompletions = vi.fn(async (model: unknown): Promise> => { - observedModelIds.push((model as { id: string }).id); - return { ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'gpt-5.4', headers: new Headers() }; - }); - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'gpt-5.4', callChatCompletions }); - queueResolution([candidate]); - - const payload = makePayload({ model: 'cop/gpt-5.4' }); - const result = await chatCompletionsServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - - // Resolver and caller payload retain the prefixed address while dispatch - // uses the catalog id. - assertEquals(lastResolveCall.model, 'cop/gpt-5.4'); - assertEquals(observedModelIds, ['gpt-5.4']); - assertEquals(payload.model, 'cop/gpt-5.4'); -}); - -test('alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - installRepo(); - // The resolver walks the alias's targets, finds no candidates for any, - // and returns empty candidates + sawModel=false. Serve renders that as - // a regular model-missing 404 with the alias name in the wording. - queueResolution([], { sawModel: false }); - - const result = await chatCompletionsServe.generate({ - payload: makePayload({ model: 'gpt-fast' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assertEquals(body.error.message, 'Model gpt-fast is not available on any configured upstream.'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini/attempt_test.ts deleted file mode 100644 index 15bbec9c28..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini/attempt_test.ts +++ /dev/null @@ -1,274 +0,0 @@ -import { test, vi } from 'vitest'; - -import { geminiAttempt } from '../../../../src/data-plane/chat/gemini/attempt.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload } from '@floway-dev/protocols/gemini'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { type MessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderResponsesResult, type ProviderStreamResult, type ResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; - -const API_KEY_ID = 'key_gemini_attempt_test'; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): GeminiPayload => ({ - contents: [{ role: 'user', parts: [{ text: 'hello' }] }], - ...overrides, -}); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeMessagesEvents = (): readonly MessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id: 'msg_1', type: 'message', role: 'assistant', content: [], - model: 'test-model', stop_reason: null, stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const makeResponsesResultEvent = (id = 'resp_test'): ResponsesStreamEvent => { - const response: ResponsesResult = { - id, object: 'response', model: 'test-model', status: 'completed', - output: [{ - type: 'message', id: 'msg_resp', role: 'assistant', status: 'completed', - content: [{ type: 'output_text', text: 'hi from responses', annotations: [] }], - }], - output_text: 'hi from responses', error: null, incomplete_details: null, - }; - return { type: 'response.completed', sequence_number: 0, response }; -}; - -const makeChatCompletionsEvents = (): readonly ChatCompletionsStreamEvent[] => [ - { - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }, - { - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, -]; - -const makeCandidate = (overrides: { - upstream?: string; - endpoints?: ModelEndpoints; - callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: MessagesUpstreamCallOptions) => Promise>; - callResponses?: (model: unknown, body: unknown, action: ResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: MessagesUpstreamCallOptions) => Promise; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ - callMessages: overrides.callMessages, - callResponses: overrides.callResponses, - callChatCompletions: overrides.callChatCompletions, - callMessagesCountTokens: overrides.callMessagesCountTokens, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, - }, - model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -test('generate translates through Chat Completions when targetApi is chat-completions', async () => { - installRepo(); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'k', headers: new Headers(), - })); - const result = await geminiAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callChatCompletions }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callChatCompletions.mock.calls.length, 1); -}); - -test('generate translates through Messages when targetApi is messages', async () => { - installRepo(); - let callOptions: MessagesUpstreamCallOptions | undefined; - const callMessages = vi.fn(async (_model, _body, _signal, opts): Promise> => { - callOptions = opts; - return { ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers() }; - }); - const result = await geminiAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessages, endpoints: { messages: {} } }), - headers: new Headers({ 'anthropic-beta': 'must-not-cross-source-protocols' }), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callMessages.mock.calls.length, 1); - assertEquals(callOptions?.anthropicBeta, []); - assertEquals(callOptions?.headers.has('anthropic-beta'), false); -}); - -test('generate translates through Responses when targetApi is responses', async () => { - installRepo(); - const callResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, events: makeProtocolFrames([makeResponsesResultEvent()]), modelKey: 'k', headers: new Headers(), - })); - const result = await geminiAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callResponses, endpoints: { responses: {} } }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callResponses.mock.calls.length, 1); -}); - -test('countTokens translates Gemini to Messages count_tokens and reshapes to totalTokens envelope', async () => { - installRepo(); - let upstreamBody: Record | undefined; - let callOptions: MessagesUpstreamCallOptions | undefined; - const callMessagesCountTokens = vi.fn(async (_model, body, _signal, opts): Promise => { - upstreamBody = body as Record; - callOptions = opts; - return { - response: new Response(JSON.stringify({ input_tokens: 42 }), { - status: 200, - headers: new Headers({ 'content-type': 'application/json' }), - }), - modelKey: 'k', - }; - }); - const result = await geminiAttempt.countTokens({ - payload: makePayload({ systemInstruction: { parts: [{ text: 'system' }] } }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessagesCountTokens }), - headers: new Headers({ 'anthropic-beta': 'must-not-cross-source-protocols' }), - }); - - assertEquals(result.type, 'plain'); - if (result.type !== 'plain') throw new Error('unreachable'); - assertEquals(result.status, 200); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body, { totalTokens: 42 }); - assertEquals(callMessagesCountTokens.mock.calls.length, 1); - assertEquals(callOptions?.anthropicBeta, []); - assertEquals(callOptions?.headers.has('anthropic-beta'), false); - // The Messages count_tokens body should never carry the translation-time - // `stream: true` flag — that field belongs to the streaming path only. - if (upstreamBody === undefined) throw new Error('upstreamBody not captured'); - assertEquals('stream' in upstreamBody, false); -}); - -test('countTokens accepts the upstream total_tokens dialect and refuses unknown shapes with a 502', async () => { - installRepo(); - const totalTokensResp = await geminiAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callMessagesCountTokens: async () => ({ - response: new Response(JSON.stringify({ total_tokens: 19 }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), - modelKey: 'k', - }), - }), - headers: new Headers(), - }); - assertEquals(totalTokensResp.type, 'plain'); - if (totalTokensResp.type !== 'plain') throw new Error('unreachable'); - assertEquals(JSON.parse(new TextDecoder().decode(totalTokensResp.body)), { totalTokens: 19 }); - - const unexpectedResp = await geminiAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callMessagesCountTokens: async () => ({ - response: new Response(JSON.stringify({ unexpected: true }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), - modelKey: 'k', - }), - }), - headers: new Headers(), - }); - assertEquals(unexpectedResp.type, 'plain'); - if (unexpectedResp.type !== 'plain') throw new Error('unreachable'); - assertEquals(unexpectedResp.status, 502); - const body = JSON.parse(new TextDecoder().decode(unexpectedResp.body)); - assertEquals(body.error.code, 502); - assertEquals(body.error.status, 'UNAVAILABLE'); -}); - -test('countTokens refuses a non-messages candidate', async () => { - installRepo(); - let thrown: unknown = null; - try { - await geminiAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ endpoints: { responses: {} } }), - headers: new Headers(), - }); - } catch (error) { - thrown = error; - } - if (!(thrown instanceof Error)) throw new Error('expected an Error to be thrown'); - assertEquals(thrown.message.includes('chatTargetPicker.pick'), true); -}); - -test('generate propagates upstream response headers through the chat-completions translation', async () => { - installRepo(); - const upstreamHeaders = new Headers({ - 'anthropic-ratelimit-unified-status': 'allowed', - 'x-request-id': 'req_gemini_xyz', - }); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'k', headers: upstreamHeaders, - })); - const result = await geminiAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callChatCompletions }), - headers: new Headers(), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - assertEquals(result.headers?.get('anthropic-ratelimit-unified-status'), 'allowed'); - assertEquals(result.headers?.get('x-request-id'), 'req_gemini_xyz'); - await collectEvents(result.events); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-safety-settings_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-safety-settings_test.ts deleted file mode 100644 index 6a1ffcc52c..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-safety-settings_test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { test } from 'vitest'; - -import { stripSafetySettings } from '../../../../../src/data-plane/chat/gemini/interceptors/strip-safety-settings.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import { type ExecuteResult, eventResult, type GeminiInvocation } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = (payload: GeminiPayload): GeminiInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'messages', - headers: new Headers(), -}); - -test('removes safetySettings without inventing missing defaults and preserves siblings', async () => { - const input = invocation({ - cachedContent: 'cachedContents/example', - safetySettings: [ - { category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH' }, - ], - }); - - await stripSafetySettings(input, stubCtx, okEvents); - - assertEquals(input.payload, { cachedContent: 'cachedContents/example' }); -}); - -test('is a no-op when safetySettings is absent', async () => { - const input = invocation({ - contents: [{ role: 'user', parts: [{ text: 'hi' }] }], - }); - - await stripSafetySettings(input, stubCtx, okEvents); - - assertEquals(input.payload, { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields_test.ts deleted file mode 100644 index 2f7749e763..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields_test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { test } from 'vitest'; - -import { stripUnsupportedPartFields } from '../../../../../src/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import { type ExecuteResult, eventResult, type GeminiInvocation } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = (payload: GeminiPayload): GeminiInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'messages', - headers: new Headers(), -}); - -test('strips fileData, executableCode, and codeExecutionResult while preserving supported part fields', async () => { - const input = invocation({ - contents: [ - { - role: 'user', - parts: [ - { - text: 'hello', - thought: true, - thoughtSignature: 'thought-signature', - inlineData: { mimeType: 'image/png', data: 'aW1hZ2U=' }, - functionCall: { id: 'call-1', name: 'lookup', args: { query: 'docs' } }, - functionResponse: { id: 'call-1', name: 'lookup', response: { ok: true } }, - fileData: { mimeType: 'text/plain', fileUri: 'gs://bucket/file.txt' }, - executableCode: { language: 'python', code: 'print(1)' }, - codeExecutionResult: { outcome: 'OUTCOME_OK', output: '1' }, - }, - ], - }, - ], - systemInstruction: { - parts: [ - { - text: 'system', - fileData: { mimeType: 'text/plain', fileUri: 'gs://bucket/system.txt' }, - }, - ], - }, - }); - - await stripUnsupportedPartFields(input, stubCtx, okEvents); - - assertEquals(input.payload, { - contents: [ - { - role: 'user', - parts: [ - { - text: 'hello', - thought: true, - thoughtSignature: 'thought-signature', - inlineData: { mimeType: 'image/png', data: 'aW1hZ2U=' }, - functionCall: { id: 'call-1', name: 'lookup', args: { query: 'docs' } }, - functionResponse: { id: 'call-1', name: 'lookup', response: { ok: true } }, - }, - ], - }, - ], - systemInstruction: { - parts: [{ text: 'system' }], - }, - }); -}); - -test('removes parts that only contain unsupported file or code fields', async () => { - const input = invocation({ - contents: [ - { - role: 'user', - parts: [ - { fileData: { mimeType: 'text/plain', fileUri: 'gs://bucket/file.txt' } }, - { text: 'keep me' }, - { - executableCode: { language: 'python', code: 'print(1)' }, - codeExecutionResult: { outcome: 'OUTCOME_OK', output: '1' }, - }, - ], - }, - ], - systemInstruction: { - parts: [ - { fileData: { mimeType: 'text/plain', fileUri: 'gs://bucket/system.txt' } }, - { text: 'system' }, - ], - }, - }); - - await stripUnsupportedPartFields(input, stubCtx, okEvents); - - assertEquals(input.payload, { - contents: [ - { - role: 'user', - parts: [{ text: 'keep me' }], - }, - ], - systemInstruction: { - parts: [{ text: 'system' }], - }, - }); -}); - -test('is a no-op when no unsupported part fields are present', async () => { - const input = invocation({ - contents: [ - { role: 'user', parts: [{ text: 'hello' }] }, - ], - }); - - await stripUnsupportedPartFields(input, stubCtx, okEvents); - - assertEquals(input.payload, { - contents: [ - { role: 'user', parts: [{ text: 'hello' }] }, - ], - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-unsupported-tools_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-unsupported-tools_test.ts deleted file mode 100644 index db84a98237..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/strip-unsupported-tools_test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { test } from 'vitest'; - -import { stripUnsupportedTools } from '../../../../../src/data-plane/chat/gemini/interceptors/strip-unsupported-tools.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import { type ExecuteResult, eventResult, type GeminiInvocation } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = (payload: GeminiPayload): GeminiInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'messages', - headers: new Headers(), -}); - -test('strips non-functionDeclarations capabilities and drops groups that become empty', async () => { - const input = invocation({ - tools: [ - { - functionDeclarations: [ - { - name: 'lookup', - description: 'Look up a value', - parameters: { type: 'object', properties: { query: { type: 'string' } } }, - }, - ], - googleSearch: {}, - googleSearchRetrieval: {}, - codeExecution: {}, - computerUse: {}, - urlContext: {}, - fileSearch: {}, - mcpServers: [{ name: 'server' }], - googleMaps: {}, - }, - { googleSearch: {} }, - { codeExecution: {} }, - ], - }); - - await stripUnsupportedTools(input, stubCtx, okEvents); - - assertEquals(input.payload, { - tools: [ - { - functionDeclarations: [ - { - name: 'lookup', - description: 'Look up a value', - parameters: { type: 'object', properties: { query: { type: 'string' } } }, - }, - ], - }, - ], - }); -}); - -test('removes the tools field entirely when every group becomes empty', async () => { - const input = invocation({ - tools: [ - { googleSearch: {} }, - { codeExecution: {} }, - ], - }); - - await stripUnsupportedTools(input, stubCtx, okEvents); - - assertEquals(input.payload, {}); -}); - -test('is a no-op when tools is absent', async () => { - const input = invocation({ - contents: [{ role: 'user', parts: [{ text: 'hi' }] }], - }); - - await stripUnsupportedTools(input, stubCtx, okEvents); - - assertEquals(input.payload, { - contents: [{ role: 'user', parts: [{ text: 'hi' }] }], - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/suppress-thought-parts_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/suppress-thought-parts_test.ts deleted file mode 100644 index ff6eae5488..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini/interceptors/suppress-thought-parts_test.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { test } from 'vitest'; - -import { suppressThoughtParts } from '../../../../../src/data-plane/chat/gemini/interceptors/suppress-thought-parts.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import { type ExecuteResult, eventResult, type GeminiInvocation } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: GeminiPayload): GeminiInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'messages', - headers: new Headers(), -}); - -const streamingResult = (events: GeminiStreamEvent[]) => (): Promise>> => - Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - })(), - testTelemetryModelIdentity, - )); - -const collect = async (events: AsyncIterable>): Promise => { - const out: GeminiStreamEvent[] = []; - for await (const frame of events) if (frame.type === 'event') out.push(frame.event); - return out; -}; - -test('drops thought parts and suppresses candidates that become empty without a finishReason', async () => { - const upstream: GeminiStreamEvent[] = [ - { - candidates: [ - { - index: 0, - content: { - role: 'model', - parts: [ - { thought: true, text: 'reasoning...' }, - { text: 'final answer' }, - ], - }, - }, - ], - }, - { - candidates: [ - { - index: 0, - content: { role: 'model', parts: [{ thought: true, text: 'more reasoning' }] }, - }, - ], - }, - ]; - - const res = await suppressThoughtParts(invocation({}), stubCtx, streamingResult(upstream)); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), [ - { - candidates: [ - { - index: 0, - content: { role: 'model', parts: [{ text: 'final answer' }] }, - }, - ], - }, - ]); -}); - -test('preserves an empty candidate when it carries a finishReason', async () => { - const upstream: GeminiStreamEvent[] = [ - { - candidates: [ - { - index: 0, - content: { role: 'model', parts: [{ thought: true, text: 'reasoning...' }] }, - finishReason: 'STOP', - }, - ], - }, - ]; - - const res = await suppressThoughtParts(invocation({}), stubCtx, streamingResult(upstream)); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), [ - { - candidates: [ - { - index: 0, - content: { role: 'model', parts: [] }, - finishReason: 'STOP', - }, - ], - }, - ]); -}); - -test('passes thought parts through when includeThoughts is opted in', async () => { - const upstream: GeminiStreamEvent[] = [ - { - candidates: [ - { - index: 0, - content: { - role: 'model', - parts: [ - { thought: true, text: 'reasoning...' }, - { text: 'final answer' }, - ], - }, - }, - ], - }, - ]; - - const res = await suppressThoughtParts( - invocation({ generationConfig: { thinkingConfig: { includeThoughts: true } } }), - stubCtx, - streamingResult(upstream), - ); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), upstream); -}); - -test('passes error frames through unchanged', async () => { - const upstream: GeminiStreamEvent[] = [ - { error: { code: 500, message: 'oops', status: 'INTERNAL' } }, - ]; - - const res = await suppressThoughtParts(invocation({}), stubCtx, streamingResult(upstream)); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), upstream); -}); - -test('preserves usageMetadata-only frames that have no candidates', async () => { - const upstream: GeminiStreamEvent[] = [ - { usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 5, totalTokenCount: 8 } }, - ]; - - const res = await suppressThoughtParts(invocation({}), stubCtx, streamingResult(upstream)); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), upstream); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini/serve_test.ts deleted file mode 100644 index 1d24a9daa7..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini/serve_test.ts +++ /dev/null @@ -1,479 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; - -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload } from '@floway-dev/protocols/gemini'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderResponsesResult, type ProviderStreamResult, type ResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; - -// Mock the resolver seam so each test hands the serve exactly the model -// candidates it wants, optionally with an alias-rules overlay attached. -interface QueuedResolution { - readonly candidates: readonly ModelCandidate[]; - readonly sawModel: boolean; - readonly failedUpstreams: readonly string[]; -} -const resolutionsQueue: QueuedResolution[] = []; -const lastResolveCall: { model?: string } = {}; -vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { - const original = await importOriginal(); - return { - ...original, - enumerateModelCandidates: vi.fn(async ({ model }: { model: string }) => { - lastResolveCall.model = model; - const next = resolutionsQueue.shift(); - if (next === undefined) throw new Error('serve_test: no resolution enqueued'); - return next; - }), - }; -}); - -const { geminiServe } = await import('../../../../src/data-plane/chat/gemini/serve.ts'); - -const API_KEY_ID = 'key_gemini_serve_test'; - -const queueResolution = ( - candidates: readonly ModelCandidate[], - extra: { sawModel?: boolean; aliasRules?: AliasRules } = {}, -): void => { - const rules = extra.aliasRules; - resolutionsQueue.push({ - candidates: rules !== undefined ? candidates.map(c => ({ ...c, rules })) : candidates, - sawModel: extra.sawModel ?? candidates.length > 0, - failedUpstreams: [], - }); -}; - -afterEach(() => { resolutionsQueue.length = 0; }); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): GeminiPayload => ({ - contents: [{ role: 'user', parts: [{ text: 'hello' }] }], - ...overrides, -}); - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeChatCompletionsEvents = (text = 'hi'): readonly ChatCompletionsStreamEvent[] => [ - { - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: { content: text }, finish_reason: null }], - }, - { - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, -]; - -const makeMessagesEvents = (id = 'msg_1'): readonly MessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id, type: 'message', role: 'assistant', content: [], - model: 'test-model', stop_reason: null, stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const makeResponsesResultEvent = (id = 'resp_test'): ResponsesStreamEvent => { - const response: ResponsesResult = { - id, object: 'response', model: 'test-model', status: 'completed', - output: [{ - type: 'message', id: 'msg_resp', role: 'assistant', status: 'completed', - content: [{ type: 'output_text', text: 'hi from responses', annotations: [] }], - }], - output_text: 'hi from responses', error: null, incomplete_details: null, - }; - return { type: 'response.completed', sequence_number: 0, response }; -}; - -const makeCandidate = (overrides: { - upstream?: string; - targetApi?: 'chat-completions' | 'messages' | 'responses'; - endpoints?: ModelEndpoints; - callChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callResponses?: (model: unknown, body: unknown, action: ResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const targetApi = overrides.targetApi ?? 'chat-completions'; - // The gemini serve layer picks the target from model.endpoints (chat-completions - // first, then messages, then responses). Narrow endpoints to the target this - // test wants so the gemini serve layer picks the expected target via - // geminiGenerateTarget.canServe. An explicit `endpoints` override beats the - // default `targetApi`-derived map, for tests that need a candidate whose - // endpoints satisfy none of the target preferences. - const endpoints = overrides.endpoints ?? (targetApi === 'chat-completions' - ? { chatCompletions: {} } - : targetApi === 'messages' - ? { messages: {} } - : { responses: {} }); - const provider = stubProvider({ - callChatCompletions: overrides.callChatCompletions, - callMessages: overrides.callMessages, - callResponses: overrides.callResponses, - callMessagesCountTokens: overrides.callMessagesCountTokens, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, - }, - model: stubInternalModel({ endpoints }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -const expectType = (r: T, k: K): Extract => { - assertEquals(r.type, k); - return r as Extract; -}; - -test('generate translates through native Chat Completions target end to end', async () => { - installRepo(); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'k', headers: new Headers(), - })); - queueResolution([makeCandidate({ targetApi: 'chat-completions', callChatCompletions })]); - - const result = await geminiServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const events = expectType(result, 'events').events; - await collectEvents(events); - assertEquals(callChatCompletions.mock.calls.length, 1); -}); - -test('generate translates through Messages when only that endpoint is exposed', async () => { - installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers(), - })); - queueResolution([makeCandidate({ targetApi: 'messages', callMessages })]); - - const result = await geminiServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const events = expectType(result, 'events').events; - await collectEvents(events); - assertEquals(callMessages.mock.calls.length, 1); -}); - -test('generate translates through Responses when only that endpoint is exposed', async () => { - installRepo(); - const callResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, events: makeProtocolFrames([makeResponsesResultEvent()]), modelKey: 'k', headers: new Headers(), - })); - queueResolution([makeCandidate({ targetApi: 'responses', callResponses })]); - - const result = await geminiServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const events = expectType(result, 'events').events; - await collectEvents(events); - assertEquals(callResponses.mock.calls.length, 1); -}); - -test('generate falls through to the next candidate when the first yields an upstream error', async () => { - installRepo(); - const payload = makePayload({ - contents: [{ - role: 'user', - parts: [{ text: 'hello', thought: true, thoughtSignature: 'opaque-signature' }], - }], - }); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'second-key', headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', targetApi: 'chat-completions', callChatCompletions: firstCall }), - makeCandidate({ upstream: 'up_b', targetApi: 'chat-completions', callChatCompletions: secondCall }), - ]); - - const result = await geminiServe.generate({ - payload, - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - // The narrowed candidate list exists exactly so a transient upstream - // failure (5xx/429/network) on one entry rolls over to the next. The - // second candidate's success is the request's final answer. - expectType(result, 'events'); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(payload.contents?.[0]?.parts, [{ - text: 'hello', - thought: true, - thoughtSignature: 'opaque-signature', - }]); -}); - -// A mid-attempt throw (interceptor bug / translation error / provider-layer -// JS exception not represented as a ChatServeFailure) must attribute the perf -// error row to the throwing candidate, not the previous one that already -// failed cleanly with a 5xx. -test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => { - throw new Error('simulated provider-layer JS exception'); - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', targetApi: 'chat-completions', callChatCompletions: firstCall }), - makeCandidate({ upstream: 'up_b', targetApi: 'chat-completions', callChatCompletions: secondCall }), - ]); - - const ctx = makeGatewayCtx(); - await geminiServe.generate({ - payload: makePayload(), - ctx, - model: 'test-model', - headers: new Headers(), - }).then( - () => { throw new Error('expected geminiServe.generate to throw'); }, - (error: unknown) => { - assertEquals((error as Error).message, 'simulated provider-layer JS exception'); - }, - ); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(ctx.attempt.telemetry?.upstream, 'up_b'); -}); - -test('generate is a routing no-op for a bare user-text request (degenerate path)', async () => { - installRepo(); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'k', headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', targetApi: 'chat-completions', callChatCompletions }), - makeCandidate({ upstream: 'up_b', targetApi: 'chat-completions', callChatCompletions }), - ]); - - const result = await geminiServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const events = expectType(result, 'events').events; - await collectEvents(events); - assertEquals(callChatCompletions.mock.calls.length, 1); -}); - -test('generate renders model-missing as a Google RPC 404 when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await geminiServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'unknown-model', - headers: new Headers(), - }); - - const upstreamError = expectType(result, 'api-error'); - assertEquals(upstreamError.status, 404); - const body = JSON.parse(new TextDecoder().decode(upstreamError.body)); - assertEquals(body.error.code, 404); - assertEquals(body.error.status, 'NOT_FOUND'); - assert(typeof body.error.message === 'string' && body.error.message.includes('unknown-model')); -}); - -test('generate filters out candidates whose endpoints do not satisfy the gemini-generate preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callChatCompletions = vi.fn(); - // geminiGenerateTarget prefers chat-completions > messages > responses; - // an endpoints-only `completions` candidate matches none and is filtered. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { completions: {} }, callChatCompletions })]); - - const result = await geminiServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'wrong-endpoint-model', - headers: new Headers(), - }); - - const upstreamError = expectType(result, 'api-error'); - assertEquals(upstreamError.status, 400); - const body = JSON.parse(new TextDecoder().decode(upstreamError.body)); - assertEquals(body.error.code, 400); - assertEquals(body.error.status, 'INVALID_ARGUMENT'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - // Defense-in-depth: even though `pick` would also throw on this candidate's - // endpoints, the spy pins that the filter happens BEFORE pick is reached, - // not after a fallback. Sibling tests in messages/responses/chat-completions - // serve_test.ts mirror this assertion for the same reason. - assertEquals(callChatCompletions.mock.calls.length, 0); -}); - -test('countTokens translates Gemini to Messages count_tokens and returns the Gemini envelope', async () => { - installRepo(); - const callMessagesCountTokens = vi.fn(async (): Promise => ({ - response: new Response(JSON.stringify({ input_tokens: 17 }), { - status: 200, headers: new Headers({ 'content-type': 'application/json' }), - }), - modelKey: 'k', - })); - queueResolution([makeCandidate({ targetApi: 'messages', callMessagesCountTokens })]); - - const result = await geminiServe.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const plain = expectType(result, 'plain'); - assertEquals(plain.status, 200); - const body = JSON.parse(new TextDecoder().decode(plain.body)); - assertEquals(body, { totalTokens: 17 }); - assertEquals(callMessagesCountTokens.mock.calls.length, 1); -}); - -test('countTokens renders a Google RPC NOT_FOUND when no Messages-capable candidate exists', async () => { - installRepo(); - queueResolution([]); - - const result = await geminiServe.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'no-messages-model', - headers: new Headers(), - }); - - const upstreamError = expectType(result, 'api-error'); - assertEquals(upstreamError.status, 404); - const body = JSON.parse(new TextDecoder().decode(upstreamError.body)); - assertEquals(body.error.code, 404); - assertEquals(body.error.status, 'NOT_FOUND'); - assert(typeof body.error.message === 'string' && body.error.message.includes('no-messages-model is not')); -}); - -test('countTokens filters out candidates whose endpoints do not satisfy the gemini-countTokens preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callMessagesCountTokens = vi.fn(); - // geminiCountTokensTarget = chatTargetPicker(['messages']); a candidate - // exposing only chatCompletions matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { chatCompletions: {} }, callMessagesCountTokens })]); - - const result = await geminiServe.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'wrong-endpoint-model', - headers: new Headers(), - }); - - const upstreamError = expectType(result, 'api-error'); - assertEquals(upstreamError.status, 400); - const body = JSON.parse(new TextDecoder().decode(upstreamError.body)); - assertEquals(body.error.code, 400); - assertEquals(body.error.status, 'INVALID_ARGUMENT'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callMessagesCountTokens.mock.calls.length, 0); -}); - -test('alias resolution overlays rules onto the target IR at the wire call', async () => { - installRepo(); - const capturedBodies: Record[] = []; - const callChatCompletions = vi.fn(async (_model: unknown, body: unknown): Promise> => { - capturedBodies.push(body as Record); - return { ok: true, events: makeProtocolFrames(makeChatCompletionsEvents()), modelKey: 'gpt-5.4', headers: new Headers() }; - }); - queueResolution([makeCandidate({ targetApi: 'chat-completions', callChatCompletions })], { - aliasRules: { reasoning: { effort: 'high', budget_tokens: 1024 }, verbosity: 'low' }, - }); - - const result = await geminiServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'gemini-fast', - headers: new Headers(), - }); - await collectEvents(expectType(result, 'events').events); - - // The resolver sees the inbound alias id verbatim. Target-id walking - // happens inside the resolver. - assertEquals(lastResolveCall.model, 'gemini-fast'); - // Alias rules land on the terminal wire target's native slots — for a - // Gemini→ChatCompletions traversal that is `reasoning_effort` and - // `verbosity`. Rules that lack a native CC slot (budget_tokens) drop. - const observed = capturedBodies[0]!; - assertEquals(observed.reasoning_effort, 'high'); - assertEquals(observed.verbosity, 'low'); -}); - -test('alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - installRepo(); - queueResolution([], { sawModel: false }); - - const result = await geminiServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'gemini-fast', - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.message, 'Model gemini-fast is not available on any configured upstream.'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/messages/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/attempt_test.ts deleted file mode 100644 index 554753814e..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/messages/attempt_test.ts +++ /dev/null @@ -1,529 +0,0 @@ -import { test, vi } from 'vitest'; - -import { messagesAttempt } from '../../../../src/data-plane/chat/messages/attempt.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesClientTool, MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ResponsesPayload, ResponsesResult } from '@floway-dev/protocols/responses'; -import { type MessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderResponsesResult, type ProviderStreamResult, type ResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import type { FlagId } from '@floway-dev/provider/flags'; -import { assertEquals, assertExists, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; - -const API_KEY_ID = 'key_messages_attempt_test'; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): MessagesPayload => ({ - model: 'test-model', - max_tokens: 32, - messages: [{ role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeMessagesEvents = (): readonly MessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id: 'msg_1', - type: 'message', - role: 'assistant', - content: [], - model: 'test-model', - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - endpoints?: ModelEndpoints; - callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: MessagesUpstreamCallOptions) => Promise>; - callResponses?: (model: unknown, body: unknown, action: ResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: MessagesUpstreamCallOptions) => Promise; - enabledFlags?: ReadonlySet; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const endpoints = overrides.endpoints ?? { chatCompletions: {}, responses: {}, messages: {} }; - const provider = stubProvider({ - callMessages: overrides.callMessages, - callResponses: overrides.callResponses, - callChatCompletions: overrides.callChatCompletions, - callMessagesCountTokens: overrides.callMessagesCountTokens, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, - }, - model: stubInternalModel({ - endpoints, - providerModels: { - [upstream]: stubProviderModel({ endpoints, enabledFlags: new Set(overrides.enabledFlags ?? []) }), - }, - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -test('generate native messages target calls provider.callMessages with no rewrite', async () => { - installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers(), - })); - const result = await messagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessages }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callMessages.mock.calls.length, 1); -}); - -test('generate carries anthropic-beta through the Messages boundary outside the provider allowlist', async () => { - installRepo(); - let upstreamHeaders: Headers | undefined; - let upstreamAnthropicBeta: readonly string[] | undefined; - const callMessages = vi.fn(async (_model, _body, _signal, opts): Promise> => { - upstreamHeaders = opts?.headers; - upstreamAnthropicBeta = opts?.anthropicBeta; - return { ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers() }; - }); - - const result = await messagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessages }), - headers: new Headers({ - 'anthropic-beta': 'must-not-enter-ordinary-headers', - 'x-unlisted': 'discard', - }), - anthropicBeta: ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20'], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertExists(upstreamHeaders); - assertEquals(Object.fromEntries(upstreamHeaders), {}); - assertEquals(upstreamAnthropicBeta, ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20']); -}); - -test('generate translate-to-responses branch routes through responsesAttempt', async () => { - installRepo(); - let upstreamHeaders: Headers | undefined; - const respResp: ResponsesResult = { - id: 'resp_x', object: 'response', model: 'test-model', status: 'completed', - output: [{ - type: 'message', id: 'msg_resp', role: 'assistant', status: 'completed', - content: [{ type: 'output_text', text: 'hi', annotations: [] }], - }], - output_text: 'hi', error: null, incomplete_details: null, - }; - const callResponses = vi.fn(async (_model, _body, _action, _signal, opts): Promise => { - upstreamHeaders = opts?.headers; - return { - action: 'generate', ok: true, - events: makeProtocolFrames([{ type: 'response.completed', sequence_number: 0, response: respResp }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - const result = await messagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callResponses, endpoints: { responses: {} } }), - headers: new Headers({ 'anthropic-beta': 'must-not-enter-ordinary-headers' }), - anthropicBeta: ['context-1m-2025-08-07'], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callResponses.mock.calls.length, 1); - assertEquals(upstreamHeaders?.has('anthropic-beta'), false); -}); - -test('generate does not carry Messages beta metadata through translation to Chat Completions', async () => { - installRepo(); - let upstreamHeaders: Headers | undefined; - const callChatCompletions = vi.fn(async (_model, _body, _signal, opts): Promise> => { - upstreamHeaders = opts?.headers; - return { - ok: true, - events: makeProtocolFrames([{ - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - - const result = await messagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callChatCompletions, endpoints: { chatCompletions: {} } }), - headers: new Headers({ 'anthropic-beta': 'must-not-enter-ordinary-headers' }), - anthropicBeta: ['context-1m-2025-08-07'], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(upstreamHeaders?.has('anthropic-beta'), false); -}); - -test('generate lets the target system-to-developer rewrite take precedence over the source system-to-user rewrite', async () => { - installRepo(); - const observedBodies: Omit[] = []; - const callResponses = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { - action: 'generate', - ok: true, - events: makeProtocolFrames([{ - type: 'response.completed', - sequence_number: 0, - response: { - id: 'resp_x', - object: 'response', - model: 'test-model', - status: 'completed', - output: [], - output_text: '', - error: null, - incomplete_details: null, - }, - }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - - const result = await messagesAttempt.generate({ - payload: makePayload({ - messages: [ - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callResponses, - endpoints: { responses: {} }, - enabledFlags: new Set([ - 'rewrite-mid-conv-system-to-user', - 'rewrite-system-to-developer', - ]), - }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callResponses.mock.calls.length, 1); - const observedBody = observedBodies[0]; - if (!observedBody) throw new Error('expected observed Responses body'); - const input = observedBody.input; - if (!Array.isArray(input)) throw new Error('expected Responses input array'); - assertEquals(input[0], { type: 'message', role: 'user', content: 'hello' }); - assertEquals(input[1], { type: 'message', role: 'developer', content: 'inline instructions' }); -}); - -test('generate translate-to-responses branch rewrites a multi-block system prefix to developer', async () => { - installRepo(); - const observedBodies: Omit[] = []; - const callResponses = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { - action: 'generate', - ok: true, - events: makeProtocolFrames([{ - type: 'response.completed', - sequence_number: 0, - response: { - id: 'resp_x', - object: 'response', - model: 'test-model', - status: 'completed', - output: [], - output_text: '', - error: null, - incomplete_details: null, - }, - }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - - const result = await messagesAttempt.generate({ - payload: makePayload({ - system: [{ type: 'text', text: 'base A' }, { type: 'text', text: 'base B' }], - messages: [ - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callResponses, - endpoints: { responses: {} }, - enabledFlags: new Set(['rewrite-system-to-developer']), - }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callResponses.mock.calls.length, 1); - const observedBody = observedBodies[0]; - if (!observedBody) throw new Error('expected observed Responses body'); - const input = observedBody.input; - if (!Array.isArray(input)) throw new Error('expected Responses input array'); - assertEquals(input[0], { - type: 'message', - role: 'developer', - content: [{ type: 'input_text', text: 'base A' }, { type: 'input_text', text: 'base B' }], - }); - assertEquals(input[1], { type: 'message', role: 'user', content: 'hello' }); - assertEquals(input[2], { type: 'message', role: 'developer', content: 'inline instructions' }); -}); - -test('countTokens proxies the upstream response as a plain result', async () => { - installRepo(); - const callMessagesCountTokens = vi.fn(async (): Promise => ({ - response: new Response(JSON.stringify({ input_tokens: 7 }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), - modelKey: 'k', - })); - - const result = await messagesAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessagesCountTokens }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'plain'); - if (result.type !== 'plain') throw new Error('unreachable'); - assertEquals(result.status, 200); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.input_tokens, 7); - assertEquals(callMessagesCountTokens.mock.calls.length, 1); -}); - -test('countTokens carries anthropic-beta through the Messages boundary outside the provider allowlist', async () => { - installRepo(); - let upstreamHeaders: Headers | undefined; - let upstreamAnthropicBeta: readonly string[] | undefined; - const callMessagesCountTokens = vi.fn(async (_model, _body, _signal, opts): Promise => { - upstreamHeaders = opts?.headers; - upstreamAnthropicBeta = opts?.anthropicBeta; - return { response: Response.json({ input_tokens: 7 }), modelKey: 'k' }; - }); - - await messagesAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessagesCountTokens }), - headers: new Headers({ - 'anthropic-beta': 'must-not-enter-ordinary-headers', - 'x-unlisted': 'discard', - }), - anthropicBeta: ['context-1m-2025-08-07'], - }); - - assertExists(upstreamHeaders); - assertEquals(Object.fromEntries(upstreamHeaders), {}); - assertEquals(upstreamAnthropicBeta, ['context-1m-2025-08-07']); -}); - -test('countTokens applies generation request transforms before provider dispatch', async () => { - installRepo(); - const observedBodies: Array> = []; - const callMessagesCountTokens = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { response: Response.json({ input_tokens: 9 }), modelKey: 'k' }; - }); - - const result = await messagesAttempt.countTokens({ - payload: makePayload({ - system: 'x-anthropic-billing-header: token\ncch=deadbeef1234;\nbase rules', - messages: [ - { role: 'system', content: 'inline rules' }, - { role: 'user', content: 'hello' }, - ], - thinking: { type: 'enabled', budget_tokens: 1024 }, - output_config: { effort: 'high' }, - tool_choice: { type: 'tool', name: 'lookup' }, - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callMessagesCountTokens, - enabledFlags: new Set([ - 'strip-billing-attribution', - 'disable-reasoning-on-forced-tool-choice', - 'rewrite-mid-conv-system-to-user', - ]), - }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'plain'); - assertEquals(observedBodies, [{ - max_tokens: 32, - system: 'base rules', - messages: [ - { role: 'user', content: 'inline rules' }, - { role: 'user', content: 'hello' }, - ], - thinking: { type: 'disabled' }, - tool_choice: { type: 'tool', name: 'lookup' }, - }]); -}); - -test('countTokens prepares the generation web-search request shape', async () => { - installRepo(); - const observedBodies: Array> = []; - const callMessagesCountTokens = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { response: Response.json({ input_tokens: 11 }), modelKey: 'k' }; - }); - - const result = await messagesAttempt.countTokens({ - payload: makePayload({ tools: [{ type: 'web_search_20260209', max_uses: 3 }] }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callMessagesCountTokens, - enabledFlags: new Set(['messages-web-search-shim']), - }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'plain'); - const tool = observedBodies[0]?.tools?.[0] as MessagesClientTool | undefined; - if (tool === undefined) throw new Error('expected rewritten web-search tool'); - assertEquals(tool.name, 'web_search'); - assertEquals('type' in tool, false); - assertEquals(tool.input_schema, { - type: 'object', - properties: { query: { type: 'string', description: 'Search query' } }, - required: ['query'], - }); -}); - -test('countTokens refuses a non-messages candidate', async () => { - installRepo(); - let thrown: unknown = null; - try { - await messagesAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ endpoints: { responses: {} } }), - headers: new Headers(), - anthropicBeta: [], - }); - } catch (error) { - thrown = error; - } - if (!(thrown instanceof Error)) throw new Error('expected an Error to be thrown'); - assertEquals(thrown.message.includes('chatTargetPicker.pick'), true); -}); - -test('generate attaches the performance context to the result', async () => { - installRepo(); - const ctx = mockChatGatewayCtx({ - apiKeyId: API_KEY_ID, - wantsStream: true, - runtimeLocation: 'SJC', - }); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'gpt-test', headers: new Headers(), - })); - - const result = await messagesAttempt.generate({ - payload: makePayload(), - ctx, - candidate: makeCandidate({ upstream: 'up_perf', callMessages }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - // result.performance carries the full dimension set that respond.ts will - // use to record telemetry once the stream settles. - assertExists(result.performance); - assertEquals(result.performance.keyId, API_KEY_ID); - assertEquals(result.performance.model, 'test-model'); - assertEquals(result.performance.upstream, 'up_perf'); - assertEquals(result.performance.runtimeLocation, 'SJC'); - - await collectEvents(result.events); -}); - -test('generate propagates upstream response headers onto the EventResult so respond can forward them', async () => { - installRepo(); - const upstreamHeaders = new Headers({ - 'anthropic-ratelimit-unified-status': 'allowed', - 'request-id': 'req_messages_xyz', - }); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: upstreamHeaders, - })); - const result = await messagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callMessages }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - assertEquals(result.headers?.get('anthropic-ratelimit-unified-status'), 'allowed'); - assertEquals(result.headers?.get('request-id'), 'req_messages_xyz'); - await collectEvents(result.events); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/messages/interceptors/apply-role-compatibility_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/interceptors/apply-role-compatibility_test.ts deleted file mode 100644 index 536bc91612..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/messages/interceptors/apply-role-compatibility_test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { test } from 'vitest'; - -import { withRoleCompatibilityApplied } from '../../../../../src/data-plane/chat/messages/interceptors/apply-role-compatibility.ts'; -import type { MessagesInvocation } from '../../../../../src/data-plane/chat/messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesMessage, MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const gatewayCtx = mockChatGatewayCtx(); -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const applyRoles = async ( - messages: MessagesMessage[], - enabledFlags: ReadonlySet, - targetApi: MessagesInvocation['targetApi'] = 'messages', -): Promise => { - const payload: MessagesPayload = { model: 'test-model', max_tokens: 1, messages }; - const invocation: MessagesInvocation = { - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), - }; - await withRoleCompatibilityApplied(invocation, gatewayCtx, okEvents); - return invocation.payload.messages; -}; - -test('leaves roles unchanged without the flag or at a translated target', async () => { - const messages: MessagesMessage[] = [{ role: 'system', content: 'inline rules' }]; - assertEquals(await applyRoles(messages, new Set()), messages); - assertEquals( - await applyRoles(messages, new Set(['rewrite-mid-conv-system-to-user']), 'responses'), - messages, - ); -}); - -test('rewrites every inline system message and preserves content', async () => { - const content = [{ type: 'text' as const, text: 'inline rules' }]; - assertEquals( - await applyRoles( - [ - { role: 'system', content: 'first rules' }, - { role: 'user', content: 'hello' }, - { role: 'system', content }, - ], - new Set(['rewrite-mid-conv-system-to-user']), - ), - [ - { role: 'user', content: 'first rules' }, - { role: 'user', content: 'hello' }, - { role: 'user', content }, - ], - ); - const result = await applyRoles( - [{ role: 'system', content }], - new Set(['rewrite-mid-conv-system-to-user']), - ); - assert(result[0]?.content === content); -}); - -test('handles empty input and leaves non-system messages unchanged', async () => { - assertEquals(await applyRoles([], new Set(['rewrite-mid-conv-system-to-user'])), []); - const messages: MessagesMessage[] = [ - { role: 'user', content: 'hello' }, - { role: 'assistant', content: 'hi' }, - ]; - assertEquals(await applyRoles(messages, new Set(['rewrite-mid-conv-system-to-user'])), messages); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice_test.ts deleted file mode 100644 index b981516ce3..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice_test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { test } from 'vitest'; - -import { withReasoningDisabledOnForcedToolChoice } from '../../../../../src/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice.ts'; -import type { MessagesInvocation } from '../../../../../src/data-plane/chat/messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { stubModelCandidate, testTelemetryModelIdentity, assertEquals } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = ( - payload: MessagesPayload, - enabledFlags: ReadonlySet = new Set(['disable-reasoning-on-forced-tool-choice']), -): MessagesInvocation => ({ - payload, - candidate: stubModelCandidate({ - model: { endpoints: { messages: {} } }, - enabledFlags, - }), - targetApi: 'messages', - headers: new Headers(), -}); - -test('messages forced tool_choice disables thinking and strips output_config', async () => { - const input = invocation({ - model: 'm', - messages: [], - max_tokens: 1, - thinking: { type: 'enabled', budget_tokens: 1024 }, - output_config: { effort: 'high' }, - tool_choice: { type: 'tool', name: 'x' }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.thinking, { type: 'disabled' }); - assertEquals(input.payload.output_config, undefined); -}); - -test('messages any tool_choice also disables thinking', async () => { - const input = invocation({ - model: 'm', - messages: [], - max_tokens: 1, - thinking: { type: 'enabled', budget_tokens: 1024 }, - tool_choice: { type: 'any' }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.thinking, { type: 'disabled' }); -}); - -test('messages non-forced tool_choice leaves reasoning untouched', async () => { - for (const type of ['auto', 'none'] as const) { - const input = invocation({ - model: 'm', - messages: [], - max_tokens: 1, - thinking: { type: 'enabled', budget_tokens: 1024 }, - tool_choice: { type }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.thinking, { - type: 'enabled', - budget_tokens: 1024, - }); - } -}); - -test('messages forced tool_choice preserves structured-output format while stripping reasoning effort', async () => { - const schema = { type: 'object', properties: { x: { type: 'string' } }, required: ['x'], additionalProperties: false }; - const input = invocation({ - model: 'm', - messages: [], - max_tokens: 1, - thinking: { type: 'enabled', budget_tokens: 1024 }, - output_config: { effort: 'high', format: { type: 'json_schema', schema } }, - tool_choice: { type: 'tool', name: 'x' }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.thinking, { type: 'disabled' }); - assertEquals(input.payload.output_config, { format: { type: 'json_schema', schema } }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/messages/interceptors/strip-billing-attribution_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/interceptors/strip-billing-attribution_test.ts deleted file mode 100644 index 00573400f8..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/messages/interceptors/strip-billing-attribution_test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { test } from 'vitest'; - -import { stripBillingAttribution } from '../../../../../src/data-plane/chat/messages/interceptors/strip-billing-attribution.ts'; -import type { MessagesInvocation } from '../../../../../src/data-plane/chat/messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import { type ExecuteResult, eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -interface InvocationOptions { - flagOn?: boolean; -} - -const invocation = (payload: MessagesPayload, { flagOn = true }: InvocationOptions = {}): MessagesInvocation => ({ - payload, - candidate: stubModelCandidate({ - model: { endpoints: { messages: {} } }, - enabledFlags: flagOn ? new Set(['strip-billing-attribution']) : new Set(), - }), - targetApi: 'messages', - headers: new Headers(), -}); - -test('strips billing-header lines and cch hashes from a string system prompt while preserving the rest', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: 'You are a helpful assistant.\nx-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;\nKeep going.', - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, 'You are a helpful assistant.\n\n\nKeep going.'); -}); - -test('strips per-block from an array-form system prompt and filters blocks that become empty', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: [ - { type: 'text', text: 'You are a helpful assistant.' }, - { type: 'text', text: 'x-anthropic-billing-header: token\ncch=abcdef12345' }, - { type: 'text', text: 'Keep going. cch=99fffaa1;' }, - ], - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, [ - { type: 'text', text: 'You are a helpful assistant.' }, - { type: 'text', text: 'Keep going.' }, - ]); -}); - -test('deletes the system field entirely when every array block becomes empty', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: [ - { type: 'text', text: 'x-anthropic-billing-header: token' }, - { type: 'text', text: 'cch=deadbeef1234;' }, - ], - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals('system' in input.payload, false); -}); - -test('deletes a string system field that becomes empty after stripping', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: 'x-anthropic-billing-header: token\ncch=deadbeef1234;', - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals('system' in input.payload, false); -}); - -test('is a no-op when system is absent', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals('system' in input.payload, false); -}); - -test('leaves a system prompt without billing markers untouched', async () => { - const original = 'You are a helpful assistant. Respond in markdown and use code fences for snippets.'; - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: original, - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, original); -}); - -test('leaves the billing block intact when the strip flag is off (claude-code default)', async () => { - const system = 'You are a helpful assistant.\nx-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;\nKeep going.'; - const input = invocation( - { - model: 'm', - max_tokens: 1, - messages: [], - system, - }, - { flagOn: false }, - ); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, system); -}); - -test('leaves an array-form billing block intact when the strip flag is off', async () => { - const system = [ - { type: 'text' as const, text: 'You are a helpful assistant.' }, - { type: 'text' as const, text: 'x-anthropic-billing-header: token\ncch=abcdef12345' }, - { type: 'text' as const, text: 'Keep going. cch=99fffaa1;' }, - ]; - const input = invocation( - { - model: 'm', - max_tokens: 1, - messages: [], - system, - }, - { flagOn: false }, - ); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, system); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/messages/respond_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/respond_test.ts deleted file mode 100644 index 74175b1519..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/messages/respond_test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { Hono } from 'hono'; -import { test } from 'vitest'; - -import { respondMessages } from '../../../../src/data-plane/chat/messages/respond.ts'; -import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import { eventResult, type ExecuteResult } from '@floway-dev/provider'; -import { assert, assertEquals, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -// --- header forwarding --- - -const forwardedHeadersFixture = (): Headers => new Headers({ - // forwardable: vendor traces, plan billing, vendor `x-*`, arbitrary custom - 'anthropic-ratelimit-unified-status': 'allowed_warning', - 'anthropic-ratelimit-unified-fallback-percentage': '50', - 'request-id': 'req_anthropic_abc', - 'cf-ray': 'cf_ray_xyz', - 'openai-version': '2024-10-21', - 'x-custom-thing': 'ok', - // blocked: hop-by-hop, body framing, cookies. Distinctive values so we can - // tell the upstream's header from anything Hono's writers add. - 'connection': 'close', - 'transfer-encoding': 'gzip', - 'content-length': '999', - 'content-encoding': 'br', - 'content-type': 'application/x-upstream-quirk', - 'set-cookie': 'session=secret', -}); - -const makeRespondCtx = (): ChatGatewayCtx => mockChatGatewayCtx({ apiKeyId: 'key_respond_test' }); - -const messagesEventsForRespond = (): readonly MessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'claude-test', - stop_reason: null, stop_sequence: null, - usage: { input_tokens: 3, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const messagesProtocolFrames = async function* (): AsyncGenerator> { - for (const event of messagesEventsForRespond()) yield eventFrame(event); - yield doneFrame(); -}; - -const callRespond = async (wantsStream: boolean): Promise => { - initRepo(new InMemoryRepo()); - const app = new Hono(); - let captured: Response | undefined; - app.get('/', async c => { - const result: ExecuteResult> = eventResult( - messagesProtocolFrames(), - testTelemetryModelIdentity, - { headers: forwardedHeadersFixture() }, - ); - const response = await respondMessages(c, result, wantsStream, makeRespondCtx()); - captured = response; - return response; - }); - await app.request('/'); - if (!captured) throw new Error('respondMessages did not produce a Response'); - return captured; -}; - -test('respondMessages forwards upstream headers and strips hop-by-hop / framing / cookie headers on the non-streaming JSON response', async () => { - const response = await callRespond(false); - // forwarded verbatim - assertEquals(response.headers.get('anthropic-ratelimit-unified-status'), 'allowed_warning'); - assertEquals(response.headers.get('anthropic-ratelimit-unified-fallback-percentage'), '50'); - assertEquals(response.headers.get('request-id'), 'req_anthropic_abc'); - assertEquals(response.headers.get('cf-ray'), 'cf_ray_xyz'); - assertEquals(response.headers.get('openai-version'), '2024-10-21'); - assertEquals(response.headers.get('x-custom-thing'), 'ok'); - // hop-by-hop and cookies dropped - assertEquals(response.headers.get('connection'), null); - assertEquals(response.headers.get('transfer-encoding'), null); - assertEquals(response.headers.get('set-cookie'), null); - // framing headers dropped — upstream values would mis-frame the response; - // Response.json sets its own content-type, which must not echo upstream's - assertEquals(response.headers.get('content-length'), null); - assertEquals(response.headers.get('content-encoding'), null); - assertEquals(response.headers.get('content-type'), 'application/json'); -}); - -test('respondMessages forwards upstream headers and strips hop-by-hop / framing / cookie headers on the streaming SSE response', async () => { - const response = await callRespond(true); - // forwarded verbatim - assertEquals(response.headers.get('anthropic-ratelimit-unified-status'), 'allowed_warning'); - assertEquals(response.headers.get('anthropic-ratelimit-unified-fallback-percentage'), '50'); - assertEquals(response.headers.get('request-id'), 'req_anthropic_abc'); - assertEquals(response.headers.get('cf-ray'), 'cf_ray_xyz'); - assertEquals(response.headers.get('openai-version'), '2024-10-21'); - assertEquals(response.headers.get('x-custom-thing'), 'ok'); - // hop-by-hop and cookies dropped. `connection` and `transfer-encoding` - // are special-cased: Hono's streamSSE writer sets its own `keep-alive` / - // `chunked`, so we assert upstream's distinctive values did not survive - // rather than asserting absence. - assert(response.headers.get('connection') !== 'close'); - assert(response.headers.get('transfer-encoding') !== 'gzip'); - assertEquals(response.headers.get('set-cookie'), null); - // framing headers dropped; streamSSE writes its own text/event-stream and - // never emits content-length or content-encoding for a streamed body - assertEquals(response.headers.get('content-length'), null); - assertEquals(response.headers.get('content-encoding'), null); - assertEquals(response.headers.get('content-type')?.split(';')[0], 'text/event-stream'); - // Drain the body so the lazy generator releases its resources and the - // background `finally` block in `streamSSE` doesn't keep the test runner - // alive. - await response.text(); -}); - -// --- partial usage checkpointing on client disconnect --- - -// A generator whose next() resolves only when emit() supplies the next event. -// Lets a test interleave "upstream emitted frame X" with "downstream cancels", -// so the streaming finally block fires while message_stop is still in flight. diff --git a/packages/gateway/__tests__/data-plane/chat/messages/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/serve_test.ts deleted file mode 100644 index 122d4bb1aa..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/messages/serve_test.ts +++ /dev/null @@ -1,743 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; - -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { type MessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderResponsesResult, type ProviderStreamResult, type ResponsesAction, type UpstreamCallOptions, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; - -// Mock the resolver seam so each test hands the serve exactly the provider -// candidates it wants, optionally with an alias-rules overlay attached. -interface QueuedResolution { - readonly candidates: readonly ModelCandidate[]; - readonly sawModel: boolean; - readonly failedUpstreams: readonly string[]; -} -const resolutionsQueue: QueuedResolution[] = []; -const lastResolveCall: { model?: string } = {}; -vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { - const original = await importOriginal(); - return { - ...original, - enumerateModelCandidates: vi.fn(async ({ model }: { model: string }) => { - lastResolveCall.model = model; - const next = resolutionsQueue.shift(); - if (next === undefined) throw new Error('serve_test: no resolution enqueued'); - return next; - }), - }; -}); - -const { messagesServe } = await import('../../../../src/data-plane/chat/messages/serve.ts'); - -const API_KEY_ID = 'key_messages_serve_test'; - -const queueResolution = ( - candidates: readonly ModelCandidate[], - extra: { sawModel?: boolean; aliasRules?: AliasRules } = {}, -): void => { - const rules = extra.aliasRules; - resolutionsQueue.push({ - candidates: rules !== undefined ? candidates.map(c => ({ ...c, rules })) : candidates, - sawModel: extra.sawModel ?? candidates.length > 0, - failedUpstreams: [], - }); -}; - -afterEach(() => { resolutionsQueue.length = 0; }); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): MessagesPayload => ({ - model: 'test-model', - max_tokens: 32, - messages: [{ role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeMessagesResultEvents = (id = 'msg_test'): readonly MessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id, - type: 'message', - role: 'assistant', - content: [], - model: 'test-model', - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 10, output_tokens: 0 }, - }, - }, - { - type: 'content_block_start', - index: 0, - content_block: { type: 'text', text: '' }, - }, - { - type: 'content_block_delta', - index: 0, - delta: { type: 'text_delta', text: 'hi' }, - }, - { type: 'content_block_stop', index: 0 }, - { - type: 'message_delta', - delta: { stop_reason: 'end_turn', stop_sequence: null }, - usage: { output_tokens: 1 }, - }, - { type: 'message_stop' }, -]; - -const makeResponsesResultEvent = (id = 'resp_test'): ResponsesStreamEvent => { - const response: ResponsesResult = { - id, - object: 'response', - model: 'test-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_resp', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'hi from responses', annotations: [] }], - }], - output_text: 'hi from responses', - error: null, - incomplete_details: null, - }; - return { type: 'response.completed', sequence_number: 0, response }; -}; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - modelId?: string; - endpoints?: ModelEndpoints; - kind?: ModelCandidate['provider']['kind']; - enabledFlags?: ReadonlySet; - callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: MessagesUpstreamCallOptions) => Promise>; - callResponses?: (model: unknown, body: unknown, action: ResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: MessagesUpstreamCallOptions) => Promise; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const modelId = overrides.modelId ?? 'test-model'; - const kind = overrides.kind ?? 'custom'; - const provider = stubProvider({ - callMessages: overrides.callMessages, - callResponses: overrides.callResponses, - callMessagesCountTokens: overrides.callMessagesCountTokens, - }); - return { - provider: { - upstreamId: upstream, kind, name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, - }, - model: stubInternalModel({ - id: modelId, - ...(overrides.endpoints ? { endpoints: overrides.endpoints } : {}), - providerModels: { - [upstream]: stubProviderModel({ - id: modelId, - ...(overrides.endpoints ? { endpoints: overrides.endpoints } : {}), - ...(overrides.enabledFlags ? { enabledFlags: overrides.enabledFlags } : {}), - }), - }, - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -// `assertEquals(result.type, X)` does not narrow the union in the TS type -// checker; the manual `if (result.type !== X) throw` follow-up was pure -// type-narrowing scaffold. This helper asserts the variant and returns the -// narrowed value so call sites stay on a single line. -const assertResultType = ( - result: U, - type: T, -): Extract => { - assertEquals(result.type, type); - return result as Extract; -}; - -function assertIsArray(value: unknown): asserts value is readonly T[] { - assert(Array.isArray(value)); -} - -test('generate routes a native Messages candidate end to end', async () => { - installRepo(); - let callOptions: MessagesUpstreamCallOptions | undefined; - const callMessages = vi.fn(async (_model, _body, _signal, opts): Promise> => { - callOptions = opts; - return { - ok: true, - events: makeProtocolFrames(makeMessagesResultEvents()), - modelKey: 'test-model-key', - headers: new Headers(), - }; - }); - queueResolution([makeCandidate({ upstream: 'up_a', callMessages })]); - - const result = await messagesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers({ 'anthropic-beta': 'context-1m-2025-08-07, advanced-tool-use-2025-11-20' }), - }); - - const events = await collectEvents(assertResultType(result, 'events').events); - assert(events.length >= 1); - assertEquals(callMessages.mock.calls.length, 1); - assertEquals(callOptions?.anthropicBeta, ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20']); - assertEquals(callOptions?.headers.has('anthropic-beta'), false); -}); - -test('generate translates through the Responses target when only that endpoint is exposed', async () => { - installRepo(); - const callResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProtocolFrames([makeResponsesResultEvent()]), - modelKey: 'responses-model-key', - headers: new Headers(), - })); - queueResolution([makeCandidate({ upstream: 'up_r', endpoints: { responses: {} }, callResponses })]); - - const result = await messagesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - assertEquals(callResponses.mock.calls.length, 1); -}); - -test('generate falls through to the next candidate when the first yields an upstream error', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesResultEvents('msg_second')), modelKey: 'second-key', headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', callMessages: firstCall }), - makeCandidate({ upstream: 'up_b', callMessages: secondCall }), - ]); - - const result = await messagesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - // The narrowed candidate list exists exactly so a transient upstream - // failure (5xx/429/network) on one entry rolls over to the next. The - // second candidate's success is the request's final answer. - assertEquals(result.type, 'events'); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); -}); - -test('generate surfaces the last upstream error verbatim when every candidate fails', async () => { - installRepo(); - const firstError = new Response('first', { status: 503 }); - const lastError = new Response('last', { status: 502 }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callMessages: async () => ({ ok: false, response: firstError, modelKey: 'first-key' }) }), - makeCandidate({ upstream: 'up_b', callMessages: async () => ({ ok: false, response: lastError, modelKey: 'last-key' }) }), - ]); - - const result = await messagesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 502); -}); - -test('generate stops at the first candidate when the payload has no reasoning carriers to route on', async () => { - installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, - events: makeProtocolFrames(makeMessagesResultEvents()), - modelKey: 'test-model-key', - headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', callMessages }), - makeCandidate({ upstream: 'up_b', callMessages }), - ]); - - const result = await messagesServe.generate({ - payload: makePayload({ messages: [{ role: 'user', content: 'hi' }] }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - assertEquals(callMessages.mock.calls.length, 1); -}); - -test('generate renders model-missing when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await messagesServe.generate({ - payload: makePayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 404); - const body = JSON.parse(new TextDecoder().decode(failure.body)); - assertEquals(body.error.type, 'not_found_error'); - assertEquals(body.error.message, 'Model unknown-model is not available on any configured upstream.'); -}); - -test('generate filters out candidates whose endpoints do not satisfy the messages-generate preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callMessages = vi.fn(); - // messagesGenerateTarget prefers messages > responses > chat-completions; an - // endpoints-only `completions` candidate matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { completions: {} }, callMessages })]); - - const result = await messagesServe.generate({ - payload: makePayload({ model: 'wrong-endpoint-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 400); - const body = JSON.parse(new TextDecoder().decode(failure.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callMessages.mock.calls.length, 0); -}); - -test('countTokens proxies the upstream measurement response as a plain result', async () => { - installRepo(); - const observedModelIds: string[] = []; - let callOptions: MessagesUpstreamCallOptions | undefined; - const callMessagesCountTokens = vi.fn(async (model: unknown, _body, _signal, opts): Promise => { - observedModelIds.push((model as { id: string }).id); - callOptions = opts; - return { - response: new Response(JSON.stringify({ input_tokens: 42 }), { - status: 200, - headers: new Headers({ 'content-type': 'application/json' }), - }), - modelKey: 'test-model-key', - }; - }); - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'claude-target', callMessagesCountTokens }); - queueResolution([candidate]); - const payload = makePayload({ model: 'claude-alias' }); - - const result = await messagesServe.countTokens({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers({ 'anthropic-beta': 'context-1m-2025-08-07, advanced-tool-use-2025-11-20' }), - }); - - const plain = assertResultType(result, 'plain'); - assertEquals(plain.status, 200); - const body = JSON.parse(new TextDecoder().decode(plain.body)); - assertEquals(body.input_tokens, 42); - assertEquals(callMessagesCountTokens.mock.calls.length, 1); - assertEquals(callOptions?.anthropicBeta, ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20']); - assertEquals(callOptions?.headers.has('anthropic-beta'), false); - assertEquals(observedModelIds, ['claude-target']); - assertEquals(payload.model, 'claude-alias'); -}); - -test('countTokens renders model-missing as a 404 when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await messagesServe.countTokens({ - payload: makePayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 404); - const body = JSON.parse(new TextDecoder().decode(failure.body)); - assertEquals(body.error.type, 'not_found_error'); - assertEquals(body.error.message, 'Model unknown-model is not available on any configured upstream.'); -}); - -test('countTokens filters out candidates whose endpoints do not satisfy the messages-countTokens preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callMessagesCountTokens = vi.fn(); - // messagesCountTokensTarget = chatTargetPicker(['messages']); a candidate - // exposing only chatCompletions matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { chatCompletions: {} }, callMessagesCountTokens })]); - - const result = await messagesServe.countTokens({ - payload: makePayload({ model: 'wrong-endpoint-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 400); - const body = JSON.parse(new TextDecoder().decode(failure.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callMessagesCountTokens.mock.calls.length, 0); -}); - -// strip-billing-attribution defaults OFF for claude-code, so a request whose -// system prompt carries the `x-anthropic-billing-header:` block must reach -// the claude-code provider's callMessages with the block intact — otherwise -// Anthropic loses the plan-tier attribution it bills against. -test('claude-code candidate preserves x-anthropic-billing-header system block through the interceptor chain', async () => { - installRepo(); - - // Match the `strip-billing-attribution: false` decision the claude-code - // provider ships (see provider-claude-code/src/defaults.ts): the plan-tier - // block must reach Anthropic verbatim. - const claudeCodeDefaults: ReadonlySet = new Set(); - - const billingBlock = 'x-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;\ncc_entrypoint=cli'; - - const capturedBodies: Omit[] = []; - const callMessages = vi.fn(async ( - _model: unknown, - body: unknown, - ): Promise> => { - capturedBodies.push(body as Omit); - return { - ok: true, - events: makeProtocolFrames(makeMessagesResultEvents()), - modelKey: 'claude-sonnet-4-5-20250929', - }; - }); - - queueResolution([ - makeCandidate({ - upstream: 'up_cc', - kind: 'claude-code', - enabledFlags: claudeCodeDefaults, - callMessages, - }), - ]); - - const result = await messagesServe.generate({ - payload: makePayload({ - system: [ - { type: 'text', text: billingBlock }, - { type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }, - ], - }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - - assertEquals(callMessages.mock.calls.length, 1); - const observed = capturedBodies[0]!; - assertIsArray<{ text: string }>(observed.system); - assertEquals(observed.system.length, 2); - assertEquals(observed.system[0].text, billingBlock); - assertEquals(observed.system[1].text, "You are Claude Code, Anthropic's official CLI for Claude."); -}); - -// The same request routed to a copilot candidate (which carries the -// strip-billing-attribution default-on flag) must have the billing block -// stripped before the upstream call — the mirror image of the claude-code -// assertion above. -test('copilot candidate strips x-anthropic-billing-header system block via the default-on flag', async () => { - installRepo(); - - // Match `strip-billing-attribution: true` on the copilot provider default - // (see provider-copilot/src/defaults.ts). - const copilotDefaults: ReadonlySet = new Set(['strip-billing-attribution']); - - const billingBlock = 'x-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;'; - - const capturedBodies: Omit[] = []; - const callMessages = vi.fn(async ( - _model: unknown, - body: unknown, - ): Promise> => { - capturedBodies.push(body as Omit); - return { - ok: true, - events: makeProtocolFrames(makeMessagesResultEvents()), - modelKey: 'claude-sonnet-4-5', - }; - }); - - queueResolution([ - makeCandidate({ - upstream: 'up_co', - kind: 'copilot', - enabledFlags: copilotDefaults, - callMessages, - }), - ]); - - const result = await messagesServe.generate({ - payload: makePayload({ - system: [ - { type: 'text', text: billingBlock }, - { type: 'text', text: 'You are a helpful assistant.' }, - ], - }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - - assertEquals(callMessages.mock.calls.length, 1); - const observed = capturedBodies[0]!; - assertIsArray<{ text: string }>(observed.system); - assertEquals(observed.system.length, 1); - assertEquals(observed.system[0].text, 'You are a helpful assistant.'); -}); - -test('generate failover preserves billing blocks for a strip-off candidate', async () => { - installRepo(); - const billingBlock = 'x-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;'; - const system = [ - { type: 'text' as const, text: billingBlock }, - { type: 'text' as const, text: "You are Claude Code, Anthropic's official CLI for Claude." }, - ]; - const expectedSystem = structuredClone(system); - const messages = [{ role: 'user' as const, content: [{ type: 'text' as const, text: 'original user text' }] }]; - const expectedMessages = structuredClone(messages); - const firstBodies: Array> = []; - const firstCall = vi.fn(async (_model: unknown, body: unknown): Promise> => { - const firstBody = body as Omit; - firstBodies.push(firstBody); - const message = firstBody.messages[0]; - if (!Array.isArray(message.content) || message.content[0]?.type !== 'text') throw new Error('expected text content'); - message.content[0].text = 'mutated by first provider'; - return { - ok: false, - response: new Response('unavailable', { status: 503 }), - modelKey: 'first-key', - }; - }); - const observedBodies: Array> = []; - const secondCall = vi.fn(async (_model: unknown, body: unknown): Promise> => { - observedBodies.push(body as Omit); - return { - ok: true, - events: makeProtocolFrames(makeMessagesResultEvents('msg_claude_code')), - modelKey: 'claude-code-key', - }; - }); - queueResolution([ - makeCandidate({ - upstream: 'up_copilot', - kind: 'copilot', - enabledFlags: new Set(['strip-billing-attribution']), - callMessages: firstCall, - }), - makeCandidate({ - upstream: 'up_claude_code', - kind: 'claude-code', - enabledFlags: new Set(), - callMessages: secondCall, - }), - ]); - - const payload = makePayload({ system, messages }); - const result = await messagesServe.generate({ payload, ctx: makeGatewayCtx(), headers: new Headers() }); - await collectEvents(assertResultType(result, 'events').events); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(firstBodies[0]?.system, [{ type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }]); - assertEquals(observedBodies[0]?.system, expectedSystem); - assertEquals(observedBodies[0]?.messages, expectedMessages); - assertEquals(payload.system, expectedSystem); - assertEquals(payload.messages, expectedMessages); -}); - -test('countTokens failover preserves billing blocks for a strip-off candidate', async () => { - installRepo(); - const system = [ - { type: 'text' as const, text: 'x-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;' }, - { type: 'text' as const, text: 'Count this prompt.' }, - ]; - const expectedSystem = structuredClone(system); - const messages = [{ role: 'user' as const, content: [{ type: 'text' as const, text: 'original user text' }] }]; - const expectedMessages = structuredClone(messages); - const firstBodies: Array> = []; - const firstCall = vi.fn(async (_model: unknown, body: unknown): Promise => { - const firstBody = body as Omit; - firstBodies.push(firstBody); - const message = firstBody.messages[0]; - if (!Array.isArray(message.content) || message.content[0]?.type !== 'text') throw new Error('expected text content'); - message.content[0].text = 'mutated by first provider'; - return { - response: new Response('unavailable', { status: 503 }), - modelKey: 'first-key', - }; - }); - const observedBodies: Array> = []; - const secondCall = vi.fn(async (_model: unknown, body: unknown): Promise => { - observedBodies.push(body as Omit); - return { - response: Response.json({ input_tokens: 12 }), - modelKey: 'second-key', - }; - }); - queueResolution([ - makeCandidate({ - upstream: 'up_strip_on', - enabledFlags: new Set(['strip-billing-attribution']), - callMessagesCountTokens: firstCall, - }), - makeCandidate({ - upstream: 'up_strip_off', - enabledFlags: new Set(), - callMessagesCountTokens: secondCall, - }), - ]); - - const payload = makePayload({ system, messages }); - const result = await messagesServe.countTokens({ payload, ctx: makeGatewayCtx(), headers: new Headers() }); - - assertEquals(assertResultType(result, 'plain').status, 200); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(firstBodies[0]?.system, [{ type: 'text', text: 'Count this prompt.' }]); - assertEquals(observedBodies[0]?.system, expectedSystem); - assertEquals(observedBodies[0]?.messages, expectedMessages); - assertEquals(payload.system, expectedSystem); - assertEquals(payload.messages, expectedMessages); -}); - -test('alias resolution swaps the inbound model id for the target and overlays rules onto the Messages payload', async () => { - installRepo(); - const capturedBodies: MessagesPayload[] = []; - const observedModelIds: string[] = []; - const callMessages = vi.fn(async (model: unknown, body: unknown): Promise> => { - observedModelIds.push((model as { id: string }).id); - capturedBodies.push({ ...(body as Omit), model: 'claude-opus-4-7' }); - return { ok: true, events: makeProtocolFrames(makeMessagesResultEvents()), modelKey: 'claude-opus-4-7' }; - }); - // Alias flow shape: the resolver returns candidates carrying the target's - // upstream catalog id AND the alias's rule overlay on `candidate.rules`. - // The attempt stamps its private clone with `candidate.model.id` and reads - // the overlay directly off `candidate.rules` at wire-call time. - const candidate = makeCandidate({ upstream: 'up_cf', modelId: 'claude-opus-4-7', callMessages }); - queueResolution([candidate], { aliasRules: { reasoning: { effort: 'high', budget_tokens: 2048 }, serviceTier: 'fast' } }); - - const payload = makePayload({ model: 'claude-fast' }); - const result = await messagesServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - - // The resolver and caller payload retain the inbound alias while dispatch - // uses the resolved target id. - assertEquals(lastResolveCall.model, 'claude-fast'); - assertEquals(observedModelIds, ['claude-opus-4-7']); - assertEquals(payload.model, 'claude-fast'); - const observed = capturedBodies[0]!; - assertEquals(observed.output_config?.effort, 'high'); - assertEquals(observed.thinking?.budget_tokens, 2048); - // The serviceTier=fast → speed=fast bridge lands the alias rule on - // Anthropic's native Fast Mode field. - assertEquals(observed.speed, 'fast'); -}); - -test('alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - installRepo(); - queueResolution([], { sawModel: false }); - - const result = await messagesServe.generate({ - payload: makePayload({ model: 'claude-fast' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'not_found_error'); - assertEquals(body.error.message, 'Model claude-fast is not available on any configured upstream.'); -}); - -// A mid-attempt throw (interceptor bug / translation error / provider-layer JS -// exception not represented as a ChatServeFailure) must attribute the perf error -// row to the throwing candidate, not the previous one. The serve stamps -// `ctx.attempt.telemetry` synchronously in the iterateCandidates -// callback so the http.ts catch can build an internal-error result carrying -// the correct upstream, and `recordFailedRequest` lands a row rather than -// short-circuiting on missing telemetry. Passthrough's equivalent regression -// lived in passthrough-serve_test.ts, which went with that surface (R3 fix 303c4e89). -test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => { - throw new Error('simulated provider-layer JS exception'); - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callMessages: firstCall }), - makeCandidate({ upstream: 'up_b', callMessages: secondCall }), - ]); - - const ctx = makeGatewayCtx(); - await messagesServe.generate({ - payload: makePayload(), - ctx, - headers: new Headers(), - }).then( - () => { throw new Error('expected messagesServe.generate to throw'); }, - (error: unknown) => { - assertEquals((error as Error).message, 'simulated provider-layer JS exception'); - }, - ); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - // The perf attribution slot reflects the throwing upstream, so the http.ts - // catch synthesizes the internal-error result with performance context and - // the error row lands against up_b. - assertEquals(ctx.attempt.telemetry?.upstream, 'up_b'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/attempt_test.ts deleted file mode 100644 index 94434140e0..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/responses/attempt_test.ts +++ /dev/null @@ -1,664 +0,0 @@ -import { test, vi } from 'vitest'; - -import { TEST_RESPONSES_RETENTION_SECONDS, testResponsesStatePolicy } from './test-policy.ts'; -import { analyzeResponsesAffinity } from '../../../../src/data-plane/chat/responses/affinity/ingress.ts'; -import { responsesAttempt } from '../../../../src/data-plane/chat/responses/attempt.ts'; -import { hydrateResponsesPayload } from '../../../../src/data-plane/chat/responses/items/hydrate.ts'; -import * as outputModule from '../../../../src/data-plane/chat/responses/items/output.ts'; -import { createResponsesHttpStore } from '../../../../src/data-plane/chat/responses/items/store.ts'; -import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import type { StoredResponsesItem } from '../../../../src/repo/types.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import { acceptedAffinityEvaluation } from '../shared/affinity/helpers.ts'; -import { initExternalResourceFetcher } from '@floway-dev/platform'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { CanonicalResponsesPayload, ResponsesPayload, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { type MessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderModel, type ProviderResponsesResult, type ProviderStreamResult, type ResponsesAction, type UpstreamCallOptions, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; - -const API_KEY_ID = 'key_attempt_test'; - -const makeGatewayCtx = (store?: ChatGatewayCtx['store']) => - mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true, ...(store ? { store } : {}) }); - -const makePayload = (overrides: Partial = {}): CanonicalResponsesPayload => ({ - model: 'test-model', - input: [{ type: 'message', role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeResponsesResult = (id = 'resp_test'): ResponsesResult => ({ - id, - object: 'response', - model: 'test-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'hi', annotations: [] }], - }], - output_text: 'hi', - error: null, - incomplete_details: null, -}); - -const makeProviderEvents = async function* (events: readonly ResponsesStreamEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = ( - callResponses: (model: ProviderModel, body: Omit, action: ResponsesAction, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => Promise, - enabledFlags: ReadonlySet = new Set(), -): ModelCandidate => { - const provider = stubProvider({ callResponses }); - const upstream = 'up_test'; - return { - provider: { - upstreamId: upstream, - kind: 'custom', - name: upstream, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache: null, - instance: provider, - }, - model: stubInternalModel({ - providerModels: { [upstream]: stubProviderModel({ enabledFlags }) }, - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: ResponsesStreamEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -const installRepo = () => { - const repo = new InMemoryRepo(); - initRepo(repo); - void repo.apiKeys.save({ - id: API_KEY_ID, userId: 1, name: 'Responses test key', key: 'raw-responses-test', - serverSecret: '99'.repeat(32), createdAt: '2026-01-01T00:00:00.000Z', - upstreamIds: null, deletedAt: null, dumpRetentionSeconds: null, - responsesRetentionSeconds: TEST_RESPONSES_RETENTION_SECONDS, - }); - return repo; -}; - -const insertStoredItem = async (repo: InMemoryRepo, overrides: Partial & Pick & { type: string }): Promise => { - const { type, ...itemOverrides } = overrides; - const row: StoredResponsesItem = { - apiKeyId: API_KEY_ID, - itemHash: `hash-${overrides.id}`, - payload: { item: { type, id: overrides.id } }, - refreshedAt: Date.now(), - ...itemOverrides, - }; - await repo.responsesItems.insertMany([row], 0); - return row; -}; - -test('generate native success leaves source-edge state ownership to the caller', async () => { - installRepo(); - - const completedEvent: ResponsesStreamEvent = { - type: 'response.completed', - sequence_number: 0, - response: makeResponsesResult(), - }; - const callResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProviderEvents([completedEvent]), - modelKey: 'test-model-key', - headers: new Headers(), - })); - - const candidate = makeCandidate(callResponses); - const store = createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const ctx = makeGatewayCtx(store); - - const result = await responsesAttempt.generate({ - payload: makePayload(), - ctx, - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - - const events = await collectEvents(result.events); - assert(events.length >= 1, 'expected at least the response.completed event'); - - assertEquals(callResponses.mock.calls.length, 1); -}); - -test('generate isolates provider mutations with JSON-safe container cloning', async () => { - installRepo(); - const metadata = JSON.parse('{"__proto__":{"retained":true},"nested":{"value":"source"}}') as Record; - const payload = makePayload({ metadata }); - const sourceItem = payload.input[0]; - const callResponses = vi.fn(async (_model, body): Promise => { - const clonedMetadata = body.metadata as { nested: { value: string } } & Record; - assert(Object.hasOwn(clonedMetadata, '__proto__'), 'clone lost the own __proto__ JSON field'); - clonedMetadata.nested.value = 'provider'; - (body.input[0] as { role: string }).role = 'assistant'; - return { - action: 'generate', - ok: true, - events: makeProviderEvents([]), - modelKey: 'test-model-key', - }; - }); - - await responsesAttempt.generate({ - payload, - ctx: makeGatewayCtx(), - candidate: makeCandidate(callResponses), - headers: new Headers(), - }); - - assertEquals((payload.metadata as { nested: { value: string } }).nested.value, 'source'); - assertEquals((sourceItem as { role: string }).role, 'user'); -}); - -test('generate treats a translated Responses payload as opaque to native affinity and state', async () => { - installRepo(); - let observedBody: Omit | undefined; - const callResponses = vi.fn(async ( - _model: ProviderModel, - body: Omit, - ): Promise => { - observedBody = body; - return { - action: 'generate', - ok: true, - events: makeProviderEvents([{ - type: 'response.completed', - sequence_number: 0, - response: makeResponsesResult(), - }]), - modelKey: 'test-model-key', - headers: new Headers(), - }; - }); - const candidate = makeCandidate(callResponses); - const store = createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const ctx = makeGatewayCtx(store); - const carrier = await ctx.affinity.codec.wrap( - undefined, - { - upstreamId: candidate.provider.upstreamId, - modelId: candidate.model.id, - }, - 'responses.reasoning.encrypted_content', - ); - const unwrap = vi.spyOn(ctx.affinity.codec, 'unwrap'); - const getStoredItem = vi.spyOn(store, 'getItemById'); - const itemId = 'rs_source_edge'; - - const result = await responsesAttempt.generate({ - payload: makePayload({ - input: [{ type: 'reasoning', id: itemId, summary: [], encrypted_content: carrier }], - }), - ctx, - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(unwrap.mock.calls.length, 0); - assertEquals(getStoredItem.mock.calls.length, 0); - assertEquals(observedBody?.input, [{ type: 'reasoning', id: itemId, summary: [], encrypted_content: carrier }]); -}); - -test('generate applies role compatibility flags in target-chain order', async () => { - installRepo(); - let observedBody: Omit | undefined; - const callResponses = vi.fn(async ( - _model: ProviderModel, - body: Omit, - ): Promise => { - observedBody = body; - return { - action: 'generate', - ok: true, - events: makeProviderEvents([{ - type: 'response.completed', - sequence_number: 0, - response: makeResponsesResult(), - }]), - modelKey: 'test-model-key', - headers: new Headers(), - }; - }); - const candidate = makeCandidate(callResponses, new Set([ - 'rewrite-developer-to-system', - 'rewrite-mid-conv-system-to-user', - 'rewrite-system-to-developer', - ])); - - const result = await responsesAttempt.generate({ - payload: makePayload({ - input: [ - { type: 'message', role: 'system', content: 'base instructions' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), false)), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(observedBody?.input, [ - { type: 'message', role: 'system', content: 'base instructions' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'user', content: 'inline instructions' }, - ]); -}); - -test('generate defers the role rewrite until after translation to Chat Completions', async () => { - installRepo(); - let observedBody: Omit | undefined; - const callChatCompletions = vi.fn(async ( - _model: ProviderModel, - body: Omit, - ): Promise> => { - observedBody = body; - return { - ok: true, - events: (async function* () { - yield eventFrame({ - id: 'chatcmpl_test', - object: 'chat.completion.chunk', - created: 0, - model: 'test-model', - choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }], - }); - yield eventFrame({ - id: 'chatcmpl_test', - object: 'chat.completion.chunk', - created: 0, - model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }); - yield eventFrame({ - id: 'chatcmpl_test', - object: 'chat.completion.chunk', - created: 0, - model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }); - yield doneFrame(); - })(), - modelKey: 'test-model-key', - headers: new Headers(), - }; - }); - const upstream = 'up_chat'; - const endpoints = { chatCompletions: {} }; - const candidate: ModelCandidate = { - provider: { - upstreamId: upstream, - kind: 'custom', - name: upstream, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache: null, - instance: stubProvider({ callChatCompletions }), - }, - model: stubInternalModel({ - endpoints, - providerModels: { - [upstream]: stubProviderModel({ - endpoints, - enabledFlags: new Set(['rewrite-system-to-developer']), - }), - }, - }, upstream), - fetcher: directFetcher, - }; - - const result = await responsesAttempt.generate({ - payload: makePayload({ - input: [ - { type: 'message', role: 'system', content: 'base instructions' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), false)), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(observedBody?.messages, [ - { role: 'developer', content: 'base instructions' }, - { role: 'user', content: 'hello' }, - { role: 'developer', content: 'inline instructions' }, - ]); -}); - -test('generate passes non-events provider result through unchanged', async () => { - installRepo(); - const wrapSpy = vi.spyOn(outputModule, 'wrapResponsesClientOutput'); - - const upstreamResponse = new Response(JSON.stringify({ error: { message: 'nope' } }), { status: 502, headers: new Headers({ 'content-type': 'application/json' }) }); - const callResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: false, - response: upstreamResponse, - modelKey: 'test-model-key', - })); - - const candidate = makeCandidate(callResponses); - const result = await responsesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true)), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 502); - // Wrap must not run when the upstream failed before any events flowed. - assertEquals(wrapSpy.mock.calls.length, 0); - wrapSpy.mockRestore(); -}); - -test('compact returns the clean upstream result for source-edge affinity and storage', async () => { - installRepo(); - - // Native /responses/compact returns a fully-shaped compaction envelope — - // the `action: 'compact'` branch of `provider.callResponses` does the - // Copilot compaction_trigger reshape internally — so the attempt receives - // a ResponsesResult, expands it into synthetic frames, and wraps the - // output for storage. The synthesized envelope carries a `compaction` - // output item; wrap observes it and derives the 'replace' snapshot. - const compactionItem = { - type: 'compaction' as const, - id: 'cmp_1', - encrypted_content: 'ENC', - }; - const compactionResult: ResponsesResult = { - ...makeResponsesResult(), - object: 'response.compaction', - // Cast: `compaction` is an input-shaped item type the protocol's - // ResponsesResult.output type does not include but the runtime accepts. - output: [compactionItem] as unknown as ResponsesResult['output'], - }; - - const callResponses = vi.fn(async (_model: ProviderModel, _body: Omit, action: ResponsesAction): Promise => { - if (action !== 'compact') throw new Error(`compact candidate received action='${action}'`); - return { action: 'compact', ok: true, result: compactionResult, modelKey: 'test-model-key' }; - }); - - const candidate = makeCandidate(callResponses); - const store = createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const result = await responsesAttempt.invoke({ - payload: makePayload({ - input: [ - { type: 'message', role: 'user', content: 'kept message' }, - ], - }), - action: 'compact', - ctx: makeGatewayCtx(store), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'result'); - if (result.type !== 'result') throw new Error('unreachable'); - assertEquals(result.result.object, 'response.compaction'); - assertEquals(result.result.output.length, 1); - assertEquals((result.result.output[0] as { id: string }).id, 'cmp_1'); - assertEquals(result.result.id, compactionResult.id); -}); - -test('generate strips disallowed headers and injects external image loading across translation to Messages', async () => { - installRepo(); - initExternalResourceFetcher(url => { - assertEquals(url.href, 'https://example.com/image.png'); - return Promise.resolve(new Response(Uint8Array.of(1, 2, 3), { headers: { 'content-type': 'image/png' } })); - }); - let observedHeaders: Headers | undefined; - let observedAnthropicBeta: readonly string[] | undefined; - let observedBody: Omit | undefined; - const upstreamModel = stubInternalModel({ endpoints: { messages: {} } }, 'up_test'); - const messagesProvider = stubProvider({ - callMessages: async (_model, body, _signal, opts): Promise> => { - observedHeaders = opts.headers; - observedAnthropicBeta = (opts as MessagesUpstreamCallOptions).anthropicBeta; - observedBody = body as Omit; - return { - ok: true, - events: (async function* () { - yield eventFrame({ - type: 'message_start', - message: { - id: 'msg_1', type: 'message', role: 'assistant', content: [], - model: 'test-model', stop_reason: null, stop_sequence: null, - usage: { input_tokens: 1, output_tokens: 0 }, - }, - }); - yield eventFrame({ type: 'message_stop' }); - yield doneFrame(); - })(), - modelKey: 'k', - headers: new Headers(), - }; - }, - }); - const candidate: ModelCandidate = { - provider: { - upstreamId: 'up_test', kind: 'custom', name: 'up_test', inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: messagesProvider, - }, - model: upstreamModel, - fetcher: directFetcher, - }; - - const result = await responsesAttempt.generate({ - payload: makePayload({ - input: [{ - type: 'message', - role: 'user', - content: [{ type: 'input_image', image_url: 'https://example.com/image.png', detail: 'auto' }], - }], - }), - ctx: makeGatewayCtx(createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true)), - candidate, - headers: new Headers({ 'anthropic-beta': 'must-not-cross-source-protocols', 'x-test': 'abc' }), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(observedHeaders?.get('x-test'), null); - assertEquals(observedHeaders?.get('anthropic-beta'), null); - assertEquals(observedAnthropicBeta, []); - const message = observedBody?.messages[0]; - assert(message?.role === 'user' && Array.isArray(message.content)); - const image = message.content.find(block => block.type === 'image'); - assert(image?.type === 'image'); - assertEquals(image.source, { type: 'base64', media_type: 'image/png', data: 'AQID' }); -}); - -test('generate seeds privatePayload before interceptors so the web-search shim replays the prior wsc results on echo', async () => { - // End-to-end contract: when a stateless client (e.g. Codex CLI) echoes a - // prior gateway-created web_search_call by its emitted id, the web-search shim's - // `transformItems` (which runs as part of the interceptor chain) must - // find the persisted `payload.private` and emit the cached function_call - // + function_call_output pair to upstream — NOT the not-preserved - // placeholder. - // - // The wire shape we model here: - // - row.id = the public item id echoed as `wsc.id`. - // - payload.item.id = that same public id. - // - payload.private = WebSearchCallPrivatePayload (v:1, functionCallItem, ir). - // - // This regression caught a prior ordering bug where hydration + beginAttempt - // ran inside the interceptor closure, after the shim's input transform — - // so privatePayload was always empty when the shim looked it up, and - // every echoed wsc collapsed to the placeholder. - const repo = installRepo(); - const storedId = `ws_${'a'.repeat(32)}`; - const storedItem = { - type: 'web_search_call' as const, - id: storedId, - status: 'completed' as const, - action: { type: 'search' as const, query: 'deepseek v4', queries: ['deepseek v4'] }, - }; - await insertStoredItem(repo, { - id: storedId, - type: 'web_search_call', - payload: { - item: storedItem, - private: { - v: 1, - functionCallItem: { - type: 'function_call', - call_id: 'call_orig_xyz', - name: 'web_search', - arguments: '{"search_query":[{"q":"deepseek v4"}]}', - status: 'completed', - }, - ir: { - action: { type: 'search', query: 'deepseek v4', queries: ['deepseek v4'] }, - results: [{ type: 'text_result', url: 'https://example.com', title: 'Example', snippet: 'CACHED_SNIPPET_BODY' }], - }, - }, - }, - }); - - // Capture the upstream-bound body so we can verify what the shim produced - // after the echoed wsc passed through transformItems. - let capturedBody: { input?: unknown[] } | undefined; - const upstreamResponse = makeResponsesResult(); - // The shim's multi-turn loop requires `response.created` (carrying a model - // name) before any synthesized terminal envelope. Emit the canonical - // created → in_progress → completed sequence so the shim can wrap. - const upstreamEvents: ResponsesStreamEvent[] = [ - { type: 'response.created', sequence_number: 0, response: upstreamResponse }, - { type: 'response.in_progress', sequence_number: 1, response: upstreamResponse }, - { type: 'response.completed', sequence_number: 2, response: upstreamResponse }, - ]; - const callResponses = vi.fn(async (_model, body): Promise => { - capturedBody = body as { input?: unknown[] }; - return { action: 'generate', ok: true, events: makeProviderEvents(upstreamEvents), modelKey: 'test-model-key', headers: new Headers() }; - }); - const candidate = makeCandidate(callResponses, new Set(['responses-web-search-shim'])); - - const store = createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true); - await store.loadInputItems([{ type: 'web_search_call', id: storedId }], []); - const ctx = makeGatewayCtx(store); - const carrier = await ctx.affinity.codec.wrap( - undefined, - { - upstreamId: candidate.provider.upstreamId, - modelId: candidate.model.id, - }, - 'responses.reasoning.encrypted_content', - { syntheticItem: true }, - ); - - const sourcePayload = makePayload({ - input: [ - { type: 'message', role: 'user', content: 'follow-up' }, - { type: 'reasoning', id: 'rs_affinity', summary: [], encrypted_content: carrier }, - { - type: 'web_search_call', - id: storedId, - status: 'completed', - action: { type: 'search', queries: ['deepseek v4'] }, - } as unknown as never, - ], - tools: [{ type: 'web_search' }], - }); - await store.loadInputItems(sourcePayload.input, sourcePayload.input); - const hydrated = hydrateResponsesPayload(sourcePayload, store); - const affinity = await analyzeResponsesAffinity(hydrated.payload, ctx.affinity.codec); - const result = await responsesAttempt.generate({ - payload: acceptedAffinityEvaluation(affinity, candidate).materialize(), - sourceState: { - privatePayloads: hydrated.privatePayloads, - }, - ctx, - candidate, - headers: new Headers(), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - - assert(capturedBody !== undefined, 'callResponses was not invoked'); - const input = (capturedBody!.input ?? []) as Array<{ type: string; call_id?: string; output?: string; name?: string; arguments?: string }>; - // The wsc echo MUST be replaced by the recovered function_call + output pair, - // carrying the persisted call_id and the cached snippet body verbatim. - const fc = input.find(i => i.type === 'function_call' && i.call_id === 'call_orig_xyz'); - assert(fc !== undefined, 'expected replayed function_call with the persisted call_id'); - assertEquals(fc!.name, 'web_search'); - assertEquals(fc!.arguments, '{"search_query":[{"q":"deepseek v4"}]}'); - const fco = input.find(i => i.type === 'function_call_output' && i.call_id === 'call_orig_xyz'); - assert(fco !== undefined, 'expected replayed function_call_output'); - assert(fco!.output?.includes('CACHED_SNIPPET_BODY'), `expected cached body in function_call_output, got: ${fco!.output}`); - // And the not-preserved placeholder MUST NOT appear. - assert( - !input.some(i => i.type === 'function_call_output' && typeof i.output === 'string' && i.output.includes('Prior search results were not preserved')), - 'shim emitted the not-preserved placeholder despite a stored private payload', - ); -}); - -test('generate propagates upstream response headers onto the EventResult so respond can forward them', async () => { - installRepo(); - const completedEvent: ResponsesStreamEvent = { - type: 'response.completed', - sequence_number: 0, - response: makeResponsesResult(), - }; - const upstreamHeaders = new Headers({ - 'anthropic-ratelimit-unified-status': 'allowed', - 'request-id': 'req_resp_xyz', - }); - const callResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProviderEvents([completedEvent]), - modelKey: 'test-model-key', - headers: upstreamHeaders, - })); - const candidate = makeCandidate(callResponses); - const store = createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const result = await responsesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(store), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - assertEquals(result.headers?.get('anthropic-ratelimit-unified-status'), 'allowed'); - assertEquals(result.headers?.get('request-id'), 'req_resp_xyz'); - await collectEvents(result.events); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/apply-role-compatibility_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/apply-role-compatibility_test.ts deleted file mode 100644 index 2900c27e62..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/apply-role-compatibility_test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { test } from 'vitest'; - -import { withRoleCompatibilityApplied } from '../../../../../src/data-plane/chat/responses/interceptors/apply-role-compatibility.ts'; -import type { ResponsesInvocation } from '../../../../../src/data-plane/chat/responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { ResponsesInputItem } from '@floway-dev/protocols/responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const gatewayCtx = mockChatGatewayCtx(); -const okEvents = () => Promise.resolve(eventResult((async function* () { yield doneFrame(); })(), testTelemetryModelIdentity)); - -const applyRoles = async ( - input: ResponsesInputItem[], - enabledFlags: ReadonlySet, - targetApi: ResponsesInvocation['targetApi'] = 'responses', -): Promise => { - const invocation: ResponsesInvocation = { - payload: { model: 'test-model', input }, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), - action: 'generate', - }; - await withRoleCompatibilityApplied(invocation, gatewayCtx, okEvents); - return invocation.payload.input; -}; - -test('leaves roles unchanged without flags or at a translated target', async () => { - const input: ResponsesInputItem[] = [ - { type: 'message', role: 'system', content: 'rules' }, - { type: 'message', role: 'developer', content: 'developer rules' }, - ]; - - assertEquals(await applyRoles(input, new Set()), input); - assertEquals(await applyRoles(input, new Set(['rewrite-system-to-developer']), 'chat-completions'), input); -}); - -test('applies the system and developer rewrites independently', async () => { - assertEquals( - await applyRoles( - [{ type: 'message', role: 'system', content: 'rules' }], - new Set(['rewrite-system-to-developer']), - ), - [{ type: 'message', role: 'developer', content: 'rules' }], - ); - assertEquals( - await applyRoles( - [{ type: 'message', role: 'developer', content: 'rules' }], - new Set(['rewrite-developer-to-system']), - ), - [{ type: 'message', role: 'system', content: 'rules' }], - ); -}); - -test('uses non-message items as the boundary before rewriting later system', async () => { - assertEquals( - await applyRoles( - [ - { type: 'message', role: 'system', content: 'base rules' }, - { type: 'reasoning', id: 'rs_1', summary: [] }, - { type: 'message', role: 'system', content: 'inline rules' }, - ], - new Set(['rewrite-mid-conv-system-to-user']), - ), - [ - { type: 'message', role: 'system', content: 'base rules' }, - { type: 'reasoning', id: 'rs_1', summary: [] }, - { type: 'message', role: 'user', content: 'inline rules' }, - ], - ); -}); - -test('keeps a leading-only system run and an empty input unchanged', async () => { - const leading: ResponsesInputItem[] = [ - { type: 'message', role: 'system', content: 'base A' }, - { type: 'message', role: 'system', content: 'base B' }, - ]; - assertEquals(await applyRoles(leading, new Set(['rewrite-mid-conv-system-to-user'])), leading); - assertEquals(await applyRoles([], new Set(['rewrite-mid-conv-system-to-user'])), []); -}); - -test('preserves multipart content identity when rewriting a mid-conversation system message', async () => { - const content = [ - { type: 'input_text' as const, text: 'one' }, - { type: 'input_text' as const, text: 'two' }, - ]; - const result = await applyRoles( - [ - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'system', content }, - ], - new Set(['rewrite-mid-conv-system-to-user']), - ); - assertEquals(result, [ - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'user', content }, - ]); - const rewritten = result[1]; - assert(rewritten?.type === 'message' && rewritten.content === content); -}); - -test('applies overlapping flags in system-to-developer then developer-to-system order', async () => { - assertEquals( - await applyRoles( - [ - { type: 'message', role: 'system', content: 'base rules' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'system', content: 'inline rules' }, - ], - new Set([ - 'rewrite-system-to-developer', - 'rewrite-developer-to-system', - 'rewrite-mid-conv-system-to-user', - ]), - ), - [ - { type: 'message', role: 'system', content: 'base rules' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'user', content: 'inline rules' }, - ], - ); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice_test.ts deleted file mode 100644 index b56f12f84d..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice_test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { test } from 'vitest'; - -import { withReasoningDisabledOnForcedToolChoice } from '../../../../../src/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice.ts'; -import type { ResponsesInvocation } from '../../../../../src/data-plane/chat/responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => - Promise.resolve( - eventResult( - (async function* () { - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ), - ); - -const invocation = ( - payload: CanonicalResponsesPayload, - enabledFlags: ReadonlySet = new Set(['disable-reasoning-on-forced-tool-choice']), -): ResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'responses', - headers: new Headers(), - action: 'generate', -}); - -test('responses required tool_choice sets reasoning.effort to none', async () => { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'high' }, - tool_choice: 'required', - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'none' }); - const out = input.payload as unknown as Record; - assertEquals(out.thinking, undefined); - assertEquals(out.enable_thinking, undefined); -}); - -test('responses object tool_choice is forced', async () => { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'high' }, - tool_choice: { type: 'custom', name: 'x' }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'none' }); -}); - -test('responses non-forced tool_choice leaves reasoning untouched', async () => { - for (const tool_choice of ['auto', 'none'] as const) { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'high' }, - tool_choice, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'high' }); - } -}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens_test.ts deleted file mode 100644 index a8329016f3..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens_test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { expect, test } from 'vitest'; - -import { withExclusiveCachedTokensNormalized } from '../../../../../src/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens.ts'; -import type { ResponsesInvocation } from '../../../../../src/data-plane/chat/responses/interceptors/types.ts'; -import { billableUsageFromResponsesResult } from '../../../../../src/data-plane/chat/responses/usage.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = ( - enabledFlags: ReadonlySet = new Set(['usage-exclusive-cached-tokens']), - targetApi: ResponsesInvocation['targetApi'] = 'responses', -): ResponsesInvocation => ({ - payload: { model: 'test-model', input: [{ type: 'message', role: 'user', content: 'hi' }] }, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), - action: 'generate', -}); - -// Same accounting as the Charm Hyper Chat Completions capture, in Responses -// field names: 479 + 13312 + 373 = 14164, so total_tokens witnesses it. -const exclusiveUsage = (): NonNullable => ({ - input_tokens: 479, - output_tokens: 373, - total_tokens: 14164, - input_tokens_details: { cached_tokens: 13312 }, -}); - -const completedEvent = (usage: NonNullable): ResponsesStreamEvent => ({ - type: 'response.completed', - sequence_number: 0, - response: { - id: 'resp_test', - object: 'response', - model: 'test-model', - status: 'completed', - output: [], - output_text: '', - error: null, - incomplete_details: null, - usage, - }, -}); - -const run = async (ctx: ResponsesInvocation, usage: NonNullable): Promise => { - const result: ExecuteResult> = await withExclusiveCachedTokensNormalized(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame(completedEvent(usage)); - })(), - testTelemetryModelIdentity, - ))); - if (result.type !== 'events') throw new Error('expected events result'); - const frames: ProtocolFrame[] = []; - for await (const frame of result.events) frames.push(frame); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - if (!('response' in frame.event)) throw new Error('expected a response-carrying event'); - return frame.event.response; -}; - -const exclusiveUsageWithoutTotal = (): NonNullable => { - const { total_tokens: _withheld, ...rest } = exclusiveUsage(); - return rest as NonNullable; -}; - -const inclusiveUsage = (): NonNullable => ({ - input_tokens: 1000, - output_tokens: 50, - total_tokens: 1050, - input_tokens_details: { cached_tokens: 400 }, -}); - -test('folds on the totals alone, with no flag set', async () => { - const response = await run(invocation(new Set()), exclusiveUsage()); - assertEquals(response.usage?.input_tokens, 13791); - assertEquals(response.usage?.total_tokens, 14164); - assertEquals(response.usage?.input_tokens_details, { cached_tokens: 13312 }); -}); - -test('folds on the flag when the totals witness nothing', async () => { - const response = await run(invocation(), exclusiveUsageWithoutTotal()); - assertEquals(response.usage?.input_tokens, 13791); -}); - -test('leaves a normalized response billable instead of underflowing', async () => { - expect(() => billableUsageFromResponsesResult({ usage: exclusiveUsage() })).toThrowError(RangeError); - - const response = await run(invocation(new Set()), exclusiveUsage()); - assertEquals( - billableUsageFromResponsesResult(response), - { input: 479, cacheRead: 13312, cacheWrite: 0, cacheWrite1h: 0, output: 373 }, - ); -}); - -test('folds cache writes back as well', async () => { - const response = await run(invocation(), { - input_tokens: 100, - output_tokens: 10, - input_tokens_details: { cached_tokens: 120, cache_write_tokens: 80 }, - } as unknown as NonNullable); - assertEquals(response.usage?.input_tokens, 300); -}); - -test('leaves an inclusive response alone when no flag claims otherwise', async () => { - const response = await run(invocation(new Set()), inclusiveUsage()); - assertEquals(response.usage?.input_tokens, 1000); -}); - -test('raises when the flag claims exclusive and the totals say inclusive', async () => { - await expect(run(invocation(), inclusiveUsage())).rejects.toThrowError(/usage-exclusive-cached-tokens is enabled/); -}); - -test('raises naming the flag when the cache counts underflow with no verdict', async () => { - await expect(run(invocation(new Set()), exclusiveUsageWithoutTotal())) - .rejects.toThrowError(/enable usage-exclusive-cached-tokens/); -}); - -test('stands down entirely when the wire it speaks about is elsewhere', async () => { - const response = await run(invocation(undefined, 'chat-completions'), exclusiveUsageWithoutTotal()); - assertEquals(response.usage?.input_tokens, 479); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/strip-prompt-cache-key_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/strip-prompt-cache-key_test.ts deleted file mode 100644 index 192c5bcc79..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/strip-prompt-cache-key_test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { test } from 'vitest'; - -import { withPromptCacheKeyStripped } from '../../../../../src/data-plane/chat/responses/interceptors/strip-prompt-cache-key.ts'; -import type { ResponsesInvocation } from '../../../../../src/data-plane/chat/responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => - Promise.resolve( - eventResult( - (async function* () { - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ), - ); - -const invocation = ( - payload: CanonicalResponsesPayload, - enabledFlags: ReadonlySet = new Set(['strip-prompt-cache-key']), -): ResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'responses', - headers: new Headers(), - action: 'generate', -}); - -test('drops top-level prompt_cache_key when the flag is on', async () => { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - prompt_cache_key: 'thread-42', - }); - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(Object.hasOwn(input.payload, 'prompt_cache_key'), false); -}); - -test('leaves the payload untouched when prompt_cache_key is absent', async () => { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - }); - const before = input.payload; - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(input.payload, before); -}); - -test('is a no-op when the flag is not set on the candidate', async () => { - const input = invocation( - { - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - prompt_cache_key: 'thread-42', - }, - new Set(), - ); - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(input.payload.prompt_cache_key, 'thread-42'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/vendor-deepseek-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/vendor-deepseek-normalize_test.ts deleted file mode 100644 index 6d31b746b3..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/vendor-deepseek-normalize_test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { test } from 'vitest'; - -import type { ResponsesInvocation } from '../../../../../src/data-plane/chat/responses/interceptors/types.ts'; -import { withVendorDeepSeekResponsesNormalize } from '../../../../../src/data-plane/chat/responses/interceptors/vendor-deepseek-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => - Promise.resolve( - eventResult( - (async function* () { - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ), - ); - -const invocation = (payload: CanonicalResponsesPayload, enabledFlags: ReadonlySet = new Set(['vendor-deepseek'])): ResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'responses', - headers: new Headers(), - action: 'generate', -}); - -test("vendor-deepseek translates canonical reasoning.effort: 'none' into top-level thinking:{type:'disabled'}", async () => { - const input = invocation({ - model: 'deepseek-reasoner', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'none' }, - }); - - await withVendorDeepSeekResponsesNormalize(input, stubCtx, okEvents); - - const out = input.payload as unknown as Record; - assertEquals(out.reasoning, undefined); - assertEquals(out.thinking, { type: 'disabled' }); -}); - -test('vendor-deepseek leaves a real reasoning.effort value untouched (only the none sentinel triggers the rewrite)', async () => { - const input = invocation({ - model: 'deepseek-reasoner', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'high' }, - }); - - await withVendorDeepSeekResponsesNormalize(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'high' }); - const out = input.payload as unknown as Record; - assertEquals(out.thinking, undefined); -}); - -test('vendor-deepseek early-returns when its flag is not set on the candidate', async () => { - const input = invocation({ model: 'deepseek-reasoner', input: [{ type: 'message', role: 'user', content: 'hi' }], reasoning: { effort: 'none' } }, new Set()); - - await withVendorDeepSeekResponsesNormalize(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'none' }); - const out = input.payload as unknown as Record; - assertEquals(out.thinking, undefined); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/serve_test.ts deleted file mode 100644 index 3235f13987..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/responses/serve_test.ts +++ /dev/null @@ -1,652 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; - -import { TEST_RESPONSES_RETENTION_SECONDS, testResponsesStatePolicy } from './test-policy.ts'; -import { createResponsesHttpStore, MemoryStatefulResponsesBacking, LayeredStatefulResponsesStore } from '../../../../src/data-plane/chat/responses/items/store.ts'; -import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import type { StoredResponsesItem, StoredResponsesSnapshot } from '../../../../src/repo/types.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { CanonicalResponsesPayload, ResponsesPayload, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { type ModelCandidate, directFetcher, type ProviderResponsesResult, type ProviderStreamResult, type ResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; - -// Mock the resolver seam so each test hands the serve exactly the provider -// candidates it wants, optionally with an alias-rules overlay attached. -// `sawModel` defaults to true when at least one candidate was queued; the -// `model-missing` failure tests queue an empty list and expect `sawModel: -// false` so the serve renders 404 rather than 400. -interface QueuedResolution { - readonly candidates: readonly ModelCandidate[]; - readonly sawModel: boolean; - readonly failedUpstreams: readonly string[]; -} -const resolutionsQueue: QueuedResolution[] = []; -const lastResolveCall: { model?: string } = {}; -vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { - const original = await importOriginal(); - return { - ...original, - enumerateModelCandidates: vi.fn(async ({ model }: { model: string }) => { - lastResolveCall.model = model; - const next = resolutionsQueue.shift(); - if (next === undefined) throw new Error('serve_test: no resolution enqueued'); - return next; - }), - }; -}); - -const { responsesServe } = await import('../../../../src/data-plane/chat/responses/serve.ts'); -const { expandPreviousResponseId } = await import('../../../../src/data-plane/chat/responses/serve-prep.ts'); - -const API_KEY_ID = 'key_serve_test'; - -const queueResolution = ( - candidates: readonly ModelCandidate[], - extra: { sawModel?: boolean; aliasRules?: AliasRules } = {}, -): void => { - const rules = extra.aliasRules; - resolutionsQueue.push({ - candidates: rules !== undefined ? candidates.map(c => ({ ...c, rules })) : candidates, - sawModel: extra.sawModel ?? candidates.length > 0, - failedUpstreams: [], - }); -}; - -afterEach(() => { resolutionsQueue.length = 0; }); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - void repo.apiKeys.save({ - id: API_KEY_ID, userId: 1, name: 'Responses test key', key: 'raw-responses-test', - serverSecret: '99'.repeat(32), createdAt: '2026-01-01T00:00:00.000Z', - upstreamIds: null, deletedAt: null, dumpRetentionSeconds: null, - responsesRetentionSeconds: TEST_RESPONSES_RETENTION_SECONDS, - }); - return repo; -}; - -const makeGatewayCtx = (store?: ChatGatewayCtx['store']) => - mockChatGatewayCtx({ - apiKeyId: API_KEY_ID, - wantsStream: true, - store: store ?? createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true), - }); - -const makePayload = (overrides: Partial = {}): CanonicalResponsesPayload => ({ - model: 'test-model', - input: [{ type: 'message', role: 'user', content: 'hello' }], - ...overrides, -}); - -// Compact tests need a real input array (a bare string can't carry the -// compaction trigger or item_reference shapes the routing layer cares -// about). Default to the kept-user-message the existing happy-path test -// uses; override `input` when a test needs a different shape. -const compactPayload = (overrides: Partial = {}): CanonicalResponsesPayload => - makePayload({ input: [{ type: 'message', role: 'user', content: 'kept' }], ...overrides }); - -const makeResponsesResult = (id = 'resp_test'): ResponsesResult => ({ - id, - object: 'response', - model: 'test-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'hi', annotations: [] }], - }], - output_text: 'hi', - error: null, - incomplete_details: null, -}); - -const makeProtocolFrames = async function* (events: readonly E[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - modelId?: string; - endpoints?: ModelEndpoints; - callResponses?: (model: unknown, body: unknown, action: ResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ - callResponses: overrides.callResponses, - callMessages: overrides.callMessages, - callChatCompletions: overrides.callChatCompletions, - }); - return { - provider: { - upstreamId: upstream, - kind: 'custom', - name: upstream, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache: null, - instance: provider, - }, - // Default keeps stubInternalModel's three-endpoint map intact; tests that - // need a rejected candidate pass an explicit `endpoints` override. - model: stubInternalModel({ - id: overrides.modelId ?? 'test-model', - ...(overrides.endpoints ? { endpoints: overrides.endpoints } : {}), - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: ResponsesStreamEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -test('generate routes a native Responses candidate end to end', async () => { - installRepo(); - const completed: ResponsesStreamEvent = { - type: 'response.completed', - sequence_number: 0, - response: makeResponsesResult(), - }; - const callResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProtocolFrames([completed]), - modelKey: 'test-model-key', - headers: new Headers(), - })); - const candidate = makeCandidate({ upstream: 'up_a', callResponses }); - queueResolution([candidate]); - - const result = await responsesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - const events = await collectEvents(result.events); - assert(events.length >= 1); - assertEquals(callResponses.mock.calls.length, 1); -}); - -test('compact returns a result envelope from the wrapped attempt', async () => { - installRepo(); - const compactionItem = { type: 'compaction' as const, id: 'cmp_1', encrypted_content: 'ENC' }; - const compactionResult: ResponsesResult = { - ...makeResponsesResult(), - object: 'response.compaction', - output: [compactionItem] as unknown as ResponsesResult['output'], - usage: { input_tokens: 12, output_tokens: 3, total_tokens: 15 }, - }; - const observedModelIds: string[] = []; - const callResponses = vi.fn(async (model: unknown, _body: unknown, action: ResponsesAction): Promise => { - if (action !== 'compact') throw new Error(`expected compact, got ${action}`); - observedModelIds.push((model as { id: string }).id); - return { action: 'compact', ok: true, result: compactionResult, modelKey: 'test-model-key' }; - }); - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'gpt-target', callResponses }); - queueResolution([candidate]); - const payload = compactPayload({ model: 'gpt-alias' }); - - const result = await responsesServe.compact({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'result'); - if (result.type !== 'result') throw new Error('unreachable'); - assertEquals(result.result.object, 'response.compaction'); - assertEquals(callResponses.mock.calls.length, 1); - assertEquals(callResponses.mock.calls[0][2], 'compact'); - assertEquals(observedModelIds, ['gpt-target']); - assertEquals(payload.model, 'gpt-alias'); -}); - -test('generate falls through to the next candidate when the first yields an upstream error', async () => { - installRepo(); - const originalImageUrl = 'data:image/png;base64,AQID'; - const payload = makePayload({ - input: [{ - type: 'message', - role: 'user', - content: [{ type: 'input_image', image_url: originalImageUrl, detail: 'auto' }], - }], - }); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (_model: unknown, body: unknown): Promise => { - const item = (body as Omit).input[0]; - if (item.type !== 'message' || !Array.isArray(item.content) || item.content[0]?.type !== 'input_image') throw new Error('expected image content'); - item.content[0].image_url = 'data:image/webp;base64,COMPRESSED'; - return { action: 'generate', ok: false, response: firstError, modelKey: 'first-key' }; - }); - const completed: ResponsesStreamEvent = { - type: 'response.completed', - sequence_number: 0, - response: makeResponsesResult('resp_second'), - }; - let fallbackImageUrl: string | null | undefined; - const secondCall = vi.fn(async (_model: unknown, body: unknown): Promise => { - const item = (body as Omit).input[0]; - if (item.type !== 'message' || !Array.isArray(item.content) || item.content[0]?.type !== 'input_image') throw new Error('expected image content'); - fallbackImageUrl = item.content[0].image_url; - return { action: 'generate', ok: true, events: makeProtocolFrames([completed]), modelKey: 'second-key', headers: new Headers() }; - }); - const first = makeCandidate({ upstream: 'up_a', callResponses: firstCall }); - const second = makeCandidate({ upstream: 'up_b', callResponses: secondCall }); - queueResolution([first, second]); - - const result = await responsesServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - // The narrowed candidate list exists exactly so a transient upstream - // failure (5xx/429/network) on one entry rolls over to the next. The - // second candidate's success is the request's final answer. - assertEquals(result.type, 'events'); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(fallbackImageUrl, originalImageUrl); - const sourceItem = payload.input[0]; - if (sourceItem.type !== 'message' || !Array.isArray(sourceItem.content) || sourceItem.content[0]?.type !== 'input_image') throw new Error('expected source image content'); - assertEquals(sourceItem.content[0].image_url, originalImageUrl); -}); - -// A mid-attempt throw (interceptor bug / translation error / provider-layer -// JS exception not represented as a ChatServeFailure) must attribute the perf -// error row to the throwing candidate, not the previous one that already -// failed cleanly with a 5xx. -test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise => ({ - action: 'generate', ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise => { - throw new Error('simulated provider-layer JS exception'); - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callResponses: firstCall }), - makeCandidate({ upstream: 'up_b', callResponses: secondCall }), - ]); - - const ctx = makeGatewayCtx(); - await responsesServe.generate({ - payload: makePayload(), - ctx, - headers: new Headers(), - }).then( - () => { throw new Error('expected responsesServe.generate to throw'); }, - (error: unknown) => { - assertEquals((error as Error).message, 'simulated provider-layer JS exception'); - }, - ); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(ctx.attempt.telemetry?.upstream, 'up_b'); -}); - -test('generate renders model-missing when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await responsesServe.generate({ - payload: makePayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assertEquals(body.error.message, 'Model unknown-model is not available on any configured upstream.'); -}); - -test('generate filters out candidates whose endpoints do not satisfy the responses preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callResponses = vi.fn(); - // responsesTarget prefers responses > messages > chat-completions; an - // endpoints-only `completions` candidate matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { completions: {} }, callResponses })]); - - const result = await responsesServe.generate({ - payload: makePayload({ model: 'wrong-endpoint-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 400); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callResponses.mock.calls.length, 0); -}); - -test('compact renders model-missing as a 404 when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await responsesServe.compact({ - payload: compactPayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('not available')); -}); - -test('compact renders model-unsupported as a 400 when the only candidate\'s endpoints don\'t satisfy responses target preferences', async () => { - installRepo(); - const callResponses = vi.fn(); - // responsesTarget prefers responses > messages > chat-completions; an - // endpoints-only `completions` candidate matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { completions: {} }, callResponses })]); - - const result = await responsesServe.compact({ - payload: compactPayload({ model: 'wrong-endpoint-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 400); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callResponses.mock.calls.length, 0); -}); - -test('expandPreviousResponseId prepends snapshot items and strips the previous_response_id field', async () => { - const repo = installRepo(); - const previousMessageId = 'msg_previous'; - await repo.responsesItems.insertMany([{ - id: previousMessageId, - apiKeyId: API_KEY_ID, - itemHash: 'previous-message-hash', - payload: { item: { type: 'message', id: previousMessageId, role: 'user', content: 'first turn' } }, - refreshedAt: Date.now(), - }], 0); - const snapshot: StoredResponsesSnapshot = { - id: 'resp_prev', - apiKeyId: API_KEY_ID, - itemIds: [previousMessageId], - refreshedAt: Date.now(), - }; - await repo.responsesSnapshots.insert(snapshot); - - const store = createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const expanded = await expandPreviousResponseId( - makePayload({ - previous_response_id: 'resp_prev', - input: [{ type: 'message', role: 'user', content: 'second turn' }], - }), - store, - ); - - assertEquals(expanded.previous_response_id, undefined); - assertEquals(expanded.input.length, 2); - assertEquals(expanded.input[0], { type: 'item_reference', id: previousMessageId }); - assertEquals(expanded.input[1], { type: 'message', role: 'user', content: 'second turn' }); -}); - -// In-memory store backed by the layered implementation but with no repo -// behind it, so an `expandPreviousResponseId` test can sit on a snapshot -// that lives nowhere else. -const memoryStore = async (snapshots: readonly StoredResponsesSnapshot[], items: readonly StoredResponsesItem[]) => { - const backing = new MemoryStatefulResponsesBacking(); - for (const item of items) await backing.insertItems([item]); - for (const snapshot of snapshots) await backing.insertSnapshot(snapshot); - return new LayeredStatefulResponsesStore({ - apiKeyId: API_KEY_ID, - reads: [backing], - writes: [backing], - }); -}; - -test('expandPreviousResponseId resolves snapshots from a non-repo-backed store', async () => { - installRepo(); // affinity lookups in the wider flow still need a repo, but here the helper only touches the store. - const id = 'msg_memory'; - const item: StoredResponsesItem = { - id, - apiKeyId: API_KEY_ID, - itemHash: 'memory-message-hash', - payload: { item: { type: 'message', id, role: 'user', content: 'remembered' } }, - refreshedAt: 1_000, - }; - const snapshot: StoredResponsesSnapshot = { - id: 'resp_mem', - apiKeyId: API_KEY_ID, - itemIds: [id], - refreshedAt: 1_000, - }; - const store = await memoryStore([snapshot], [item]); - - const expanded = await expandPreviousResponseId( - makePayload({ previous_response_id: 'resp_mem', input: [{ type: 'message', role: 'user', content: 'new turn' }] }), - store, - ); - - assertEquals(expanded.input.length, 2); - assertEquals(expanded.input[0], { type: 'item_reference', id }); -}); - -test('generate falls through translate-out to messages target', async () => { - installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, - events: makeProtocolFrames([ - { - type: 'message_start', - message: { - id: 'msg_translated', - type: 'message', - role: 'assistant', - content: [], - model: 'test-model', - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, - ]), - modelKey: 'messages-key', - headers: new Headers(), - })); - const candidate = makeCandidate({ upstream: 'up_m', endpoints: { messages: {} }, callMessages }); - queueResolution([candidate]); - - const result = await responsesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callMessages.mock.calls.length, 1); -}); - -test('Messages biology refusal becomes a non-retryable Codex Responses policy failure', async () => { - installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, - events: makeProtocolFrames([ - { - type: 'message_start', - message: { - id: 'msg_bio_refusal', - type: 'message', - role: 'assistant', - content: [], - model: 'claude-fable-5', - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { - type: 'message_delta', - delta: { - stop_reason: 'refusal', - stop_details: { - type: 'refusal', - category: 'bio', - explanation: 'This request could enable biological harm.', - }, - stop_sequence: null, - }, - usage: { output_tokens: 0 }, - }, - { type: 'message_stop' }, - ]), - modelKey: 'messages-key', - headers: new Headers(), - })); - const candidate = makeCandidate({ upstream: 'up_m', endpoints: { messages: {} }, callMessages }); - queueResolution([candidate]); - - const result = await responsesServe.generate({ payload: makePayload(), ctx: makeGatewayCtx(), headers: new Headers() }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - const events = await collectEvents(result.events); - const failed = events.find((event): event is Extract => event.type === 'response.failed'); - - assertEquals(failed?.response.status, 'failed'); - assertEquals(failed?.response.error, { - code: 'bio_policy', - message: 'This content was flagged for possible biological risk. This request could enable biological harm.', - }); - assertEquals(callMessages.mock.calls.length, 1); -}); - -test('generate falls through translate-out to chat-completions target', async () => { - installRepo(); - const callChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, - events: makeProtocolFrames([ - { - id: 'chatcmpl_translated', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }], - }, - { - id: 'chatcmpl_translated', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }, - { - id: 'chatcmpl_translated', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, - { - id: 'chatcmpl_translated', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [], usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 }, - }, - ]), - modelKey: 'chat-completions-key', - headers: new Headers(), - })); - const candidate = makeCandidate({ upstream: 'up_c', endpoints: { chatCompletions: {} }, callChatCompletions }); - queueResolution([candidate]); - - const result = await responsesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callChatCompletions.mock.calls.length, 1); -}); - -test('alias resolution swaps the inbound model id for the target and overlays rules onto the Responses IR', async () => { - installRepo(); - const capturedBodies: ResponsesPayload[] = []; - const observedModelIds: string[] = []; - const callResponses = vi.fn(async (model: unknown, body: unknown): Promise => { - observedModelIds.push((model as { id: string }).id); - capturedBodies.push(body as ResponsesPayload); - return { action: 'generate', ok: true, events: makeProtocolFrames([{ type: 'response.completed', sequence_number: 0, response: makeResponsesResult() }]), modelKey: 'gpt-5.4', headers: new Headers() }; - }); - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'gpt-5.4', callResponses }); - queueResolution([candidate], { - aliasRules: { reasoning: { effort: 'high', summary: 'detailed' }, verbosity: 'medium', serviceTier: 'priority' }, - }); - - const payload = makePayload({ model: 'gpt-fast' }); - const result = await responsesServe.generate({ - payload, - ctx: makeGatewayCtx(createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true)), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - - // Resolver and caller payload retain the inbound alias; the provider model - // argument carries the resolved target id while the body omits `model`. - assertEquals(lastResolveCall.model, 'gpt-fast'); - assertEquals(observedModelIds, ['gpt-5.4']); - assertEquals(payload.model, 'gpt-fast'); - const observed = capturedBodies[0]!; - assertEquals(observed.reasoning?.effort, 'high'); - assertEquals(observed.reasoning?.summary, 'detailed'); - assertEquals(observed.text?.verbosity, 'medium'); - assertEquals(observed.service_tier, 'priority'); -}); - -test('alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - installRepo(); - queueResolution([], { sawModel: false }); - - const result = await responsesServe.generate({ - payload: makePayload({ model: 'gpt-fast' }), - ctx: makeGatewayCtx(createResponsesHttpStore(testResponsesStatePolicy(API_KEY_ID), Date.now(), true)), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.message, 'Model gpt-fast is not available on any configured upstream.'); -}); diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 82f0b762e1..97fabe82bb 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -9,8 +9,11 @@ import { describe, expect, it } from 'vitest'; import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/pipeline.ts'; import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; +import { geminiCountTokensPipeline } from '../../../src/data-plane/chat/gemini/count-tokens.ts'; import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; +import { messagesCountTokensPipeline } from '../../../src/data-plane/chat/messages/count-tokens.ts'; import { messagesServePipeline } from '../../../src/data-plane/chat/messages/pipeline.ts'; +import { responsesCompactPipeline } from '../../../src/data-plane/chat/responses/compact.ts'; import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; @@ -29,6 +32,9 @@ const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] ['messages', () => messagesServePipeline({ model: 'm', messages: [] } as never)], ['gemini', () => geminiServePipeline({ model: 'm', contents: [] } as never)], ['responses', () => responsesServePipeline({ model: 'm', input: [] } as never)], + ['messages count_tokens', () => messagesCountTokensPipeline({ model: 'm', messages: [] } as never)], + ['gemini countTokens', () => geminiCountTokensPipeline({ model: 'm', contents: [] } as never)], + ['responses compact', () => responsesCompactPipeline({ model: 'm', input: [] } as never)], ]; describe('every family assembles', () => { diff --git a/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts b/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts deleted file mode 100644 index 0d3ef826de..0000000000 --- a/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import type { GatewayCtx } from '../../../src/data-plane/shared/gateway-ctx.ts'; -import { iterateCandidates } from '../../../src/data-plane/shared/iterate-candidates.ts'; -import { mockGatewayCtx } from '../../test-utils/gateway-ctx.ts'; -import type { ModelCandidate, PerformanceTelemetryContext } from '@floway-dev/provider'; -import { mockPerfTelemetryContext, stubModelCandidate, stubProvider } from '@floway-dev/test-utils'; - -const stubCandidate = (id: string, upstream = 'up'): ModelCandidate => - stubModelCandidate({ - model: { id }, - provider: { - upstreamId: upstream, - kind: 'custom', - name: upstream, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache: null, - instance: stubProvider(), - }, - }); - -// iterateCandidates reads `attempt`, `apiKeyId`, and `runtimeLocation` from -// ctx — the last two feed `upstreamPerformanceContext` when stamping -// `attempt.telemetry`. Building through mockGatewayCtx keeps the stub aligned -// with the real GatewayCtx shape. -const stubCtx = (attempt: GatewayCtx['attempt']): GatewayCtx => mockGatewayCtx({ attempt }); - -describe('iterateCandidates', () => { - it('clears the timing slots and stamps telemetry from the current candidate on entry', async () => { - const attempt = { upstreamCallStartedAt: 999, firstOutputTokenAt: 999, telemetry: mockPerfTelemetryContext({ upstream: 'carryover' }) as PerformanceTelemetryContext | undefined }; - const ctx = stubCtx(attempt); - const observed: Array<{ upstreamCallStartedAt: number | null; firstOutputTokenAt: number | null; upstream: string | undefined }> = []; - - await iterateCandidates( - [stubCandidate('a', 'up_a'), stubCandidate('b', 'up_b'), stubCandidate('c', 'up_c')], - 'test', - ctx, - 'chat', - async candidate => { - observed.push({ - upstreamCallStartedAt: attempt.upstreamCallStartedAt, - firstOutputTokenAt: attempt.firstOutputTokenAt, - upstream: attempt.telemetry?.upstream, - }); - // Simulate an attempt that stamps timing then fails, so the loop - // advances to the next candidate. - attempt.upstreamCallStartedAt = 100; - attempt.firstOutputTokenAt = 200; - return candidate.model.id === 'c' - ? { type: 'events' as const } - : { type: 'api-error' as const }; - }, - ); - - // Every attempt must observe cleared timing slots on entry — a prior - // candidate's stamps (or the caller-supplied carryover) must not - // survive into the next attempt. Telemetry must reflect the CURRENT - // candidate: regressing this reintroduces the mid-attempt-throw - // misattribution the stamp was hoisted to prevent. - expect(observed).toEqual([ - { upstreamCallStartedAt: null, firstOutputTokenAt: null, upstream: 'up_a' }, - { upstreamCallStartedAt: null, firstOutputTokenAt: null, upstream: 'up_b' }, - { upstreamCallStartedAt: null, firstOutputTokenAt: null, upstream: 'up_c' }, - ]); - }); - - it('returns the first success and stops iterating', async () => { - const ctx = stubCtx({ upstreamCallStartedAt: null, firstOutputTokenAt: null, telemetry: undefined }); - let calls = 0; - const result = await iterateCandidates( - [stubCandidate('a'), stubCandidate('b'), stubCandidate('c')], - 'test', - ctx, - 'chat', - async () => { - calls++; - return { type: 'events' as const }; - }, - ); - - expect(result).toEqual({ type: 'events' }); - expect(calls).toBe(1); - }); - - it('returns the last failure once every candidate errors', async () => { - const ctx = stubCtx({ upstreamCallStartedAt: null, firstOutputTokenAt: null, telemetry: undefined }); - let index = 0; - const failures = [ - { type: 'api-error' as const, marker: 'first' }, - { type: 'internal-error' as const, marker: 'last' }, - ]; - const result = await iterateCandidates( - [stubCandidate('a'), stubCandidate('b')], - 'test', - ctx, - 'chat', - async () => failures[index++]!, - ); - - expect(result).toEqual(failures[1]); - }); - - it('treats non-2xx plain results as failure so the next candidate runs', async () => { - const ctx = stubCtx({ upstreamCallStartedAt: null, firstOutputTokenAt: null, telemetry: undefined }); - const attempts: number[] = []; - const result = await iterateCandidates( - [stubCandidate('a'), stubCandidate('b')], - 'test', - ctx, - 'chat', - async candidate => { - attempts.push(attempts.length); - return candidate.model.id === 'a' - ? { type: 'plain' as const, status: 500 } - : { type: 'plain' as const, status: 200 }; - }, - ); - - expect(attempts).toEqual([0, 1]); - expect(result).toEqual({ type: 'plain', status: 200 }); - }); -}); diff --git a/packages/gateway/src/data-plane/chat/chat-completions/attempt.ts b/packages/gateway/src/data-plane/chat/chat-completions/attempt.ts deleted file mode 100644 index 8013461f53..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/attempt.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { chatCompletionsInterceptors } from './interceptors/index.ts'; -import type { ChatCompletionsInvocation } from './interceptors/types.ts'; -import { billableUsageFromChatCompletionsEvent } from './usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; -import { messagesAttempt } from '../messages/attempt.ts'; -import { responsesAttempt } from '../responses/attempt.ts'; -import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; -import { createExternalImageLoader } from '../shared/external-image-loader.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerStreamResultToExecuteResult } from '../shared/provider-stream-result.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ModelCandidate, type ExecuteResult, providerModelOf } from '@floway-dev/provider'; -import { translateChatCompletionsViaMessages, translateChatCompletionsViaResponses } from '@floway-dev/translate'; - -// `/v1/chat/completions` generate prefers a native Chat Completions target, -// then the translated Messages path, then the translated Responses path. -export const chatCompletionsTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); - -export interface ChatCompletionsAttemptArgs { - readonly payload: ChatCompletionsPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export const chatCompletionsAttempt = { - generate: async (args: ChatCompletionsAttemptArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - const targetApi = chatCompletionsTarget.pick(candidate.model.endpoints); - const invocation: ChatCompletionsInvocation = { - payload, - candidate, - targetApi, - headers, - }; - return await runInterceptors(invocation, ctx, chatCompletionsInterceptors, async () => { - if (targetApi === 'chat-completions') { - if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const providerResult = await candidate.provider.instance.callChatCompletions( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildUpstreamCallOptions(candidate, ctx, invocation.headers), - ); - return await providerStreamResultToExecuteResult(providerResult, candidate, 'chat-completions', ctx, billableUsageFromChatCompletionsEvent); - } - if (targetApi === 'messages') { - return await traverseTranslation( - invocation.payload, - p => translateChatCompletionsViaMessages(p, { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - loadRemoteImage: createExternalImageLoader(ctx.abortSignal), - }), - translated => messagesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }), - ); - } - if (targetApi === 'responses') { - return await traverseTranslation( - invocation.payload, - p => translateChatCompletionsViaResponses(p, { model: candidate.model.id }), - translated => responsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - throw new Error(`chatCompletionsAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/errors.ts b/packages/gateway/src/data-plane/chat/chat-completions/errors.ts deleted file mode 100644 index 7944f6b24b..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/errors.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { appendFailedUpstreams } from '../../shared/failed-upstreams.ts'; -import { openAiErrorResult, type ChatServeFailure } from '../shared/errors.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult, PerformanceTelemetryContext } from '@floway-dev/provider'; -import type { TranslatorInputError } from '@floway-dev/translate'; - -// Translator surfaced a caller-input violation. Render as a 400 -// invalid_request_error so the caller sees a protocol-shaped failure -// instead of the internal-error 502 envelope. `param` falls back to -// `messages` (the Chat Completions canonical messages field) when the -// translator did not carry a more specific path. `performance` carries the -// throwing candidate's telemetry attribution when the throw fired -// mid-attempt (see AttemptState.telemetry). -export const translatorInputErrorResult = ( - error: TranslatorInputError, - performance?: PerformanceTelemetryContext, -): ExecuteResult> => - openAiErrorResult(400, error.message, { param: error.param ?? 'messages', code: null }, performance); - -export const renderChatCompletionsFailure = ( - failure: ChatServeFailure, -): ExecuteResult> => { - switch (failure.kind) { - case 'routing-unavailable': - return openAiErrorResult(400, failure.message, { param: 'input', code: 'responses_item_routing_unavailable' }); - case 'model-missing': - return openAiErrorResult(404, appendFailedUpstreams(`Model ${failure.model} is not available on any configured upstream.`, failure.failedUpstreams)); - case 'model-unsupported': - return openAiErrorResult(400, appendFailedUpstreams(`Model ${failure.model} does not support the /chat/completions endpoint.`, failure.failedUpstreams)); - } -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/apply-role-compatibility.ts deleted file mode 100644 index 84a128e9b7..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/apply-role-compatibility.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { ChatCompletionsInterceptor } from './types.ts'; -import type { ChatCompletionsMessage } from '@floway-dev/protocols/chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withRoleCompatibilityApplied: ChatCompletionsInterceptor = (ctx, _gatewayCtx, run) => { - if (ctx.targetApi !== 'chat-completions') return run(); - - const flags = providerModelOf(ctx.candidate).enabledFlags; - const rewriteSystemToDeveloper = flags.has('rewrite-system-to-developer'); - const rewriteDeveloperToSystem = flags.has('rewrite-developer-to-system'); - const rewriteMidConvSystemToUser = flags.has('rewrite-mid-conv-system-to-user'); - if (!rewriteSystemToDeveloper && !rewriteDeveloperToSystem && !rewriteMidConvSystemToUser) return run(); - - let crossedLeadingSystemRun = false; - ctx.payload = { - ...ctx.payload, - messages: ctx.payload.messages.map(message => { - let mapped: ChatCompletionsMessage = message; - if (rewriteSystemToDeveloper && mapped.role === 'system') mapped = { ...mapped, role: 'developer' }; - if (rewriteDeveloperToSystem && mapped.role === 'developer') mapped = { ...mapped, role: 'system' }; - if (!crossedLeadingSystemRun && mapped.role !== 'system') crossedLeadingSystemRun = true; - if (rewriteMidConvSystemToUser && crossedLeadingSystemRun && mapped.role === 'system') { - mapped = { ...mapped, role: 'user' }; - } - return mapped; - }), - }; - - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts deleted file mode 100644 index 234e823997..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts +++ /dev/null @@ -1,24 +0,0 @@ - -import type { ChatCompletionsInterceptor } from './types.ts'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams where forced `tool_choice` and enabled -// reasoning do not compose. Sets the gateway's canonical "no reasoning" -// sentinel `reasoning_effort: 'none'` (also OpenAI's documented disable -// value). Any active Vendor: * flag's last-running normalizer then -// translates that into the vendor's wire form (DeepSeek -// `thinking: {type:'disabled'}`, Qwen `enable_thinking: false`, etc.). -const hasForcedToolChoice = (payload: ChatCompletionsPayload): boolean => { - const toolChoice = payload.tool_choice; - if (toolChoice === undefined || toolChoice === null) return false; - if (typeof toolChoice === 'string') return toolChoice === 'required'; - return true; -}; - -export const withReasoningDisabledOnForcedToolChoice: ChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('disable-reasoning-on-forced-tool-choice')) return await run(); - if (!hasForcedToolChoice(ctx.payload)) return await run(); - ctx.payload = { ...ctx.payload, reasoning_effort: 'none' }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/include-usage-stream-options.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/include-usage-stream-options.ts deleted file mode 100644 index bbe39e2e90..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/include-usage-stream-options.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { ChatCompletionsInterceptor } from './types.ts'; - -// Chat Completions streaming only includes the final usage-only chunk when -// `stream_options.include_usage` is enabled. We force that on here because -// the gateway's source responders and usage tracking rely on those usage -// frames for both streaming passthrough and non-stream reassembly. -// -// The follow-on question is whether to surface the synthesized usage chunk to -// the client. The client's intent is whatever the caller put on -// `stream_options.include_usage` BEFORE this interceptor mutated it; the -// downstream SSE renderer reads that intent directly off the original payload -// the http entry parsed (see `chatCompletionsHttp`), so this interceptor only -// has to flip the upstream-facing flag and never needs to thread the original -// value through a Hono context slot. -// -// Reference: https://platform.openai.com/docs/api-reference/chat/create -export const withUsageStreamOptionsIncluded: ChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - ctx.payload.stream_options = ctx.payload.stream_options ? { ...ctx.payload.stream_options, include_usage: true } : { include_usage: true }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/index.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/index.ts deleted file mode 100644 index c03c469fcc..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/index.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { withRoleCompatibilityApplied } from './apply-role-compatibility.ts'; -import { withReasoningDisabledOnForcedToolChoice } from './disable-reasoning-on-forced-tool-choice.ts'; -import { withUsageStreamOptionsIncluded } from './include-usage-stream-options.ts'; -import { withExclusiveCachedTokensNormalized } from './normalize-exclusive-cached-tokens.ts'; -import { withUsageNormalized } from './normalize-usage.ts'; -import { withPromptCacheKeyStripped } from './strip-prompt-cache-key.ts'; -import type { ChatCompletionsInterceptor } from './types.ts'; -import { withVendorDeepSeekChatCompletionsNormalize } from './vendor-deepseek-normalize.ts'; -import { withVendorKimiChatCompletionsNormalize } from './vendor-kimi-normalize.ts'; -import { withVendorQwenChatCompletionsNormalize } from './vendor-qwen-normalize.ts'; - -// Unified Chat Completions interceptor list. All entries are attached to -// every candidate; each interceptor's body decides whether to act (flag-gated -// entries early-return on `providerModelOf(ctx.candidate).enabledFlags.has(flagId)`). -// -// Translated requests re-enter the selected target protocol's chain. The role -// compatibility entry therefore acts only when Chat Completions is the final -// target, after pairwise translation has finished. -// -// - withUsageStreamOptionsIncluded, withUsageNormalized: unconditional. -// Both gate the gateway's usage-tracking pipeline. Turning either off -// would silently break per-key telemetry, so neither is surfaced as a flag. -// - withReasoningDisabledOnForcedToolChoice: gated by -// `disable-reasoning-on-forced-tool-choice`. Emits the gateway's canonical -// "no reasoning" sentinel only; vendor wire form is the vendor's job. -// - withRoleCompatibilityApplied: applies role flags in the fixed order -// `system → developer → system → user`; later rewrites are authoritative -// when flags overlap, and the final step affects only mid-conversation system. -// - withPromptCacheKeyStripped: gated by `strip-prompt-cache-key`. Drops -// the top-level `prompt_cache_key` field for upstreams that reject it as -// an unknown argument (e.g. Azure DeepSeek). Runs before vendor -// normalizers so vendor-specific translation sees the already-stripped -// canonical payload. -// - withExclusiveCachedTokensNormalized: unconditional on a Chat -// Completions target. Folds the cache buckets back into `prompt_tokens` -// whenever `total_tokens` witnesses that the upstream reports them -// alongside it, and consults `usage-exclusive-cached-tokens` for the -// responses whose totals witness nothing — so the flag is a declaration -// input rather than a gate. Registered before the vendor normalizers so -// that on the inbound path it runs after them, on usage whose cache fields -// already carry OpenAI's names. -// - withVendor*ChatCompletionsNormalize: gated by `vendor-`. Registered -// LAST so that on the outbound path each gets the final say on the wire -// body and on the inbound path each gets the first say on the upstream -// stream — the generic interceptors above only see OpenAI-canonical form. -// Vendor flags are mutually exclusive in practice, but the interceptors -// are independent and run in declared order if more than one is somehow -// enabled. -export const chatCompletionsInterceptors: readonly ChatCompletionsInterceptor[] = [ - withUsageStreamOptionsIncluded, - withUsageNormalized, - withReasoningDisabledOnForcedToolChoice, - withRoleCompatibilityApplied, - withPromptCacheKeyStripped, - withExclusiveCachedTokensNormalized, - withVendorDeepSeekChatCompletionsNormalize, - withVendorQwenChatCompletionsNormalize, - withVendorKimiChatCompletionsNormalize, -]; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens.ts deleted file mode 100644 index 7a71e83548..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/normalize-exclusive-cached-tokens.ts +++ /dev/null @@ -1,97 +0,0 @@ -// Restores OpenAI's inclusive input-token contract on upstreams that report -// the cache buckets alongside `prompt_tokens` instead of inside it. -// -// OpenAI states the subset relationship outright — "Cached tokens here are -// counted as a subset of input tokens, meaning input tokens will include -// cached and uncached tokens" -// (https://github.com/openai/openai-openapi/blob/d4fb706e6e05d4cc9f1b33ca59b6e4f3e8edd439/openapi.yaml#L51043-L51049) -// — and every downstream consumer here subtracts on that basis. Anthropic -// takes the opposite convention: `input_tokens` counts only what was neither -// read from nor written to the cache, and the three buckets sum to the real -// input (https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching). -// -// A gateway that fronts an Anthropic-shaped upstream and projects it into -// Chat Completions can carry the exclusive convention through to a wire that -// declares the inclusive one. Portkey does exactly that — it assigns -// Anthropic's `input_tokens` straight to `prompt_tokens` while summing -// `total_tokens` from all four buckets -// (https://github.com/Portkey-AI/gateway/blob/669825cbe89ee51569918b8f78a9db486fd69dd4/src/providers/anthropic/chatComplete.ts#L612-L627). -// Charm Hyper produces the same shape by subtracting the cached prefix out of -// the `prompt_tokens` it received: for one observed kimi-k3 turn it reported -// `prompt_tokens 479, cached_tokens 13312, completion_tokens 373, -// total_tokens 14164`, where 479 + 13312 + 373 = 14164. -// -// `foldsExclusiveCacheTokens` owns the decision and the two contradictions -// that must not pass silently; the `usage-exclusive-cached-tokens` flag is its -// declaration input. `total_tokens` itself is left alone: under the exclusive -// convention it already counts the real input, so the rewritten -// `prompt_tokens + completion_tokens` is what it was equal to all along. - -import type { ChatCompletionsInterceptor } from './types.ts'; -import { asJsonObject, type JsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; -import { foldsExclusiveCacheTokens } from '../../../shared/telemetry/usage.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { eventFrame } from '@floway-dev/protocols/common'; -import { providerModelOf } from '@floway-dev/provider'; - -const rewriteInboundUsage = ( - chunk: ChatCompletionsStreamEvent, - declaredExclusive: boolean, - identity: string, -): ChatCompletionsStreamEvent => { - const usage = asJsonObject(chunk.usage); - if (!usage) return chunk; - const inputTokens = readJsonNumber(usage.prompt_tokens); - const outputTokens = readJsonNumber(usage.completion_tokens); - if (inputTokens == null || outputTokens == null) return chunk; - - const details = asJsonObject(usage.prompt_tokens_details); - const cacheRead = readJsonNumber(details?.cached_tokens) ?? 0; - const cacheWrite = readJsonNumber(details?.cache_creation_input_tokens) ?? readJsonNumber(details?.cache_write_tokens) ?? 0; - if (cacheRead === 0 && cacheWrite === 0) return chunk; - - const fold = foldsExclusiveCacheTokens(declaredExclusive, { - inputTokens, - outputTokens, - totalTokens: readJsonNumber(usage.total_tokens) ?? undefined, - cacheRead, - cacheWrite, - }, identity); - if (!fold) return chunk; - - const next: JsonObject = { ...usage, prompt_tokens: inputTokens + cacheRead + cacheWrite }; - return { ...chunk, usage: next as unknown as ChatCompletionsStreamEvent['usage'] }; -}; - -export const withExclusiveCachedTokensNormalized: ChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - // Everything this entry says is about one upstream's Chat Completions wire: - // the flag it reads describes how that upstream writes its usage there, and - // the remedy its errors name is a setting for that upstream. A translated - // request re-enters its target's chain, where these events are a projection - // of some other wire — the flag answers a question about counts it does not - // describe, and telling an operator to set it would be advice that cannot - // help. Nothing is lost by standing down: a translator emits the canonical - // form, which is the one case the fold has nothing to do with. - if (ctx.targetApi !== 'chat-completions') return await run(); - - const model = providerModelOf(ctx.candidate); - const declaredExclusive = model.enabledFlags.has('usage-exclusive-cached-tokens'); - const identity = `${ctx.candidate.provider.upstreamId}/${model.id}`; - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = rewriteInboundUsage(frame.event, declaredExclusive, identity); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/normalize-usage.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/normalize-usage.ts deleted file mode 100644 index ba90688a8c..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/normalize-usage.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { ChatCompletionsInterceptor } from './types.ts'; -import { asJsonObject } from '../../../../shared/json-helpers.ts'; -import { eventFrame } from '@floway-dev/protocols/common'; - -// Spec-compliant Chat Completions usage chunk shape. The OpenAI spec puts the -// final `usage` on a `choices: []` carrier chunk -// (https://platform.openai.com/docs/api-reference/chat-streaming). Some -// upstreams have been observed to attach `usage` to the same chunk that -// carries the final delta and `finish_reason`. We strip `usage` from such a -// chunk and re-emit it on a synthesized spec-compliant carrier chunk -// immediately after, so downstream consumers can rely on the standard shape. -// -// Vendor-specific cache-token field rewrites (DeepSeek -// `prompt_cache_hit_tokens` / `prompt_cache_miss_tokens`, Kimi -// `cached_tokens`) live on each vendor's own `vendor--normalize` -// interceptor and run before this one on the response path, so by the time -// the chunk reaches us its `usage.prompt_tokens_details.cached_tokens` is -// already in the OpenAI standard shape. - -export const withUsageNormalized: ChatCompletionsInterceptor = async (_ctx, _gatewayCtx, run) => { - const result = await run(); - if (result.type !== 'events') return result; - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - - const chunk = frame.event; - const usage = asJsonObject(chunk.usage); - if (!usage || chunk.choices.length === 0) { - yield frame; - continue; - } - const { usage: chunkUsage, ...withoutUsage } = chunk; - yield eventFrame(withoutUsage); - yield eventFrame({ ...withoutUsage, choices: [], usage: chunkUsage }); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key.ts deleted file mode 100644 index bb9abd2991..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/strip-prompt-cache-key.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { ChatCompletionsInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams that reject `prompt_cache_key` as an unknown -// request argument (e.g. Azure DeepSeek returns -// `unrecognized_request_argument`). Drop the top-level field before the -// request reaches the terminal. OpenAI-native and truly OpenAI-compatible -// upstreams accept it for prefix-cache attribution, so removal only happens -// under the flag. -export const withPromptCacheKeyStripped: ChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('strip-prompt-cache-key')) return await run(); - if (ctx.payload.prompt_cache_key === undefined) return await run(); - const { prompt_cache_key: _stripped, ...rest } = ctx.payload; - ctx.payload = rest; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/types.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/types.ts deleted file mode 100644 index d6d5368047..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/types.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ChatCompletionsInvocation, ExecuteResult } from '@floway-dev/provider'; - -export type { ChatCompletionsInvocation }; - -export type ChatCompletionsInterceptor = Interceptor< - ChatCompletionsInvocation, - GatewayCtx, - ExecuteResult> ->; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize.ts deleted file mode 100644 index 308c3d9052..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-deepseek-normalize.ts +++ /dev/null @@ -1,143 +0,0 @@ -// DeepSeek wire-dialect normalizer for Chat Completions. Always-attached; -// flag-gated by `vendor-deepseek`. Runs last among the gateway's interceptors -// so it has the final say on the outbound wire body and the first say on the -// inbound stream — the gateway's generic interceptors above it deal only in -// OpenAI-canonical form. -// -// Outbound (request → upstream): -// -// - `reasoning_effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel (produced when a Messages source had `thinking: { type: -// 'disabled' }`, when a Chat source sent it literally, etc.). DeepSeek -// doesn't accept 'none' in its `reasoning_effort` enum and instead uses -// a top-level `thinking: { type: 'disabled' }` field. We strip the -// sentinel and emit the DeepSeek form. -// - Assistant messages: rewrite `reasoning_text` → `reasoning_content` (and -// synthesise `reasoning_content` from `reasoning_items.summary` when the -// newer OpenAI shape is the only thing present). DeepSeek documents only -// the scalar `reasoning_content` field and reports 400s when it is -// omitted from the assistant-message replay of a multi-turn tool-call loop. -// - `response_format: { type: 'json_schema', … }` is downgraded to -// `response_format: { type: 'json_object' }`. DeepSeek's structured-output -// API supports only `json_object`; the schema body is dropped on the floor -// rather than rejected by the upstream. -// -// Inbound (stream → client): -// -// - Each delta: rewrite `reasoning_content` → `reasoning_text` so downstream -// gateway code sees the OpenAI shape. -// - Each usage chunk: remap `prompt_cache_hit_tokens` / -// `prompt_cache_miss_tokens` into OpenAI's -// `prompt_tokens_details.cached_tokens`. The remap is computed from -// `prompt_cache_hit_tokens` alone (DeepSeek's "hit" count is the cached -// prefix length); the "miss" field is dropped. -// -// References: -// - https://api-docs.deepseek.com/zh-cn/guides/thinking_mode -// - https://api-docs.deepseek.com/guides/kv_cache -// - https://api-docs.deepseek.com/quick_start/agent_integrations/oh_my_pi - -import type { ChatCompletionsInterceptor } from './types.ts'; -import { asJsonObject, type JsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; -import type { ChatCompletionsStreamEvent, ChatCompletionsPayload, ChatCompletionsReasoningItem, ChatCompletionsMessage } from '@floway-dev/protocols/chat-completions'; -import { eventFrame } from '@floway-dev/protocols/common'; -import { providerModelOf } from '@floway-dev/provider'; - -const synthesizeFromItems = (items: ChatCompletionsReasoningItem[] | null | undefined): string | undefined => { - if (!items?.length) return undefined; - const parts = items.flatMap(item => item.summary?.map(s => s.text) ?? []); - return parts.length > 0 ? parts.join('') : undefined; -}; - -const rewriteOutboundMessage = (message: ChatCompletionsMessage): ChatCompletionsMessage => { - // `reasoning_opaque` is the OpenAI-canonical signature for cross-turn - // reasoning replay; DeepSeek doesn't accept it, so it's dropped on the - // floor when we project assistant messages onto `reasoning_content`. - const { reasoning_text, reasoning_opaque: _opaque, reasoning_items, ...rest } = message; - const text = typeof reasoning_text === 'string' ? reasoning_text : synthesizeFromItems(reasoning_items); - if (text === undefined) return rest as ChatCompletionsMessage; - return { ...rest, reasoning_content: text } as ChatCompletionsMessage; -}; - -const stripCanonicalReasoningSentinel = (payload: ChatCompletionsPayload): ChatCompletionsPayload => { - if (payload.reasoning_effort !== 'none') return payload; - const { reasoning_effort: _stripped, ...rest } = payload; - return { ...rest, thinking: { type: 'disabled' as const } } as ChatCompletionsPayload; -}; - -const downgradeJsonSchemaResponseFormat = (payload: ChatCompletionsPayload): ChatCompletionsPayload => { - const rf = payload.response_format; - if (rf?.type !== 'json_schema') return payload; - return { ...payload, response_format: { type: 'json_object' } }; -}; - -const rewriteOutboundPayload = (payload: ChatCompletionsPayload): ChatCompletionsPayload => { - const withDisable = stripCanonicalReasoningSentinel(payload); - const withResponseFormat = downgradeJsonSchemaResponseFormat(withDisable); - return { - ...withResponseFormat, - messages: withResponseFormat.messages.map(rewriteOutboundMessage), - }; -}; - -const rewriteInboundDeltas = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { - let changed = false; - const choices = chunk.choices.map(choice => { - const delta = choice.delta as ChatCompletionsStreamEvent['choices'][number]['delta'] & { reasoning_content?: unknown }; - if (typeof delta.reasoning_content !== 'string') return choice; - - const { reasoning_content, ...rest } = delta; - changed = true; - return { - ...choice, - delta: { - ...rest, - ...(delta.reasoning_text === undefined ? { reasoning_text: reasoning_content } : {}), - }, - }; - }); - return changed ? { ...chunk, choices } : chunk; -}; - -const VENDOR_CACHE_FIELDS = ['prompt_cache_hit_tokens', 'prompt_cache_miss_tokens'] as const; - -const rewriteInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { - const usage = asJsonObject(chunk.usage); - if (!usage) return chunk; - const hit = readJsonNumber(usage.prompt_cache_hit_tokens); - const hasVendorField = VENDOR_CACHE_FIELDS.some(field => usage[field] !== undefined); - if (!hasVendorField) return chunk; - - const next: JsonObject = { ...usage }; - for (const field of VENDOR_CACHE_FIELDS) delete next[field]; - if (hit != null) { - next.prompt_tokens_details = { - ...(asJsonObject(usage.prompt_tokens_details) ?? {}), - cached_tokens: hit, - }; - } - return { ...chunk, usage: next as unknown as ChatCompletionsStreamEvent['usage'] }; -}; - -export const withVendorDeepSeekChatCompletionsNormalize: ChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-deepseek')) return await run(); - - ctx.payload = rewriteOutboundPayload(ctx.payload); - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = rewriteInboundUsage(rewriteInboundDeltas(frame.event)); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize.ts deleted file mode 100644 index 9aab8e88e9..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-kimi-normalize.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Kimi (Moonshot) wire-dialect normalizer for Chat Completions. -// Always-attached; flag-gated by `vendor-kimi`. Runs last among the gateway's -// interceptors so it has the first say on the inbound stream. -// -// Outbound: nothing today — Kimi accepts the OpenAI-canonical request shape -// the gateway emits. -// -// Inbound (stream → client): -// -// - Each usage chunk: remap the flat `cached_tokens` field into OpenAI's -// `prompt_tokens_details.cached_tokens`. Generic interceptors upstream of -// this one then see the standard shape. -// -// Reference: -// - https://platform.kimi.com/docs/api/chat - -import type { ChatCompletionsInterceptor } from './types.ts'; -import { asJsonObject, type JsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { eventFrame } from '@floway-dev/protocols/common'; -import { providerModelOf } from '@floway-dev/provider'; - -const rewriteInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { - const usage = asJsonObject(chunk.usage); - if (!usage) return chunk; - const cached = readJsonNumber(usage.cached_tokens); - if (cached == null) return chunk; - - const { cached_tokens: _stripped, ...rest } = usage; - const next: JsonObject = { - ...rest, - prompt_tokens_details: { - ...(asJsonObject(usage.prompt_tokens_details) ?? {}), - cached_tokens: cached, - }, - }; - return { ...chunk, usage: next as unknown as ChatCompletionsStreamEvent['usage'] }; -}; - -export const withVendorKimiChatCompletionsNormalize: ChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-kimi')) return await run(); - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = rewriteInboundUsage(frame.event); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize.ts b/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize.ts deleted file mode 100644 index cec0fcfc63..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/interceptors/vendor-qwen-normalize.ts +++ /dev/null @@ -1,30 +0,0 @@ -// Qwen wire-dialect normalizer for Chat Completions. Always-attached; -// flag-gated by `vendor-qwen`. Runs last among the gateway's interceptors so -// it has the final say on the outbound wire body. -// -// Outbound (request → upstream): -// -// - `reasoning_effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel. Qwen doesn't accept 'none' in its `reasoning_effort` enum and -// instead uses a top-level `enable_thinking: false` field. We strip the -// sentinel and emit the Qwen form. -// -// Inbound: Qwen's response shape matches OpenAI for the fields the gateway reads. -// -// Reference: -// - https://www.alibabacloud.com/help/en/model-studio/deep-thinking - -import type { ChatCompletionsInterceptor } from './types.ts'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withVendorQwenChatCompletionsNormalize: ChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-qwen')) return await run(); - - if (ctx.payload.reasoning_effort === 'none') { - const { reasoning_effort: _stripped, ...rest } = ctx.payload; - ctx.payload = { ...rest, enable_thinking: false } as ChatCompletionsPayload; - } - - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/respond.ts b/packages/gateway/src/data-plane/chat/chat-completions/respond.ts deleted file mode 100644 index 4393dcd832..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/respond.ts +++ /dev/null @@ -1,130 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapChatCompletionsAffinityEgress } from './affinity/egress.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import { chatCompletionsProtocolFrameToSSEFrame, CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, collectChatCompletionsProtocolEventsToResult, chatCompletionsErrorPayloadMessage } from '@floway-dev/protocols/chat-completions'; -import { eventFrame, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, type PlainResult, type InternalDebugError, toInternalDebugError } from '@floway-dev/provider'; -import { apiErrorToResponse } from '@floway-dev/provider'; - -export const respondChatCompletions = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - includeUsageChunk: boolean, - ctx: GatewayCtx, -): Promise => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return apiErrorToResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return internalChatCompletionsErrorResponse(result.status, result.error); - } - - if (result.type === 'plain') { - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); - } - - const state = new SourceStreamState(); - const observed = observeChatCompletionsFrames(result.events, state, ctx); - const frames = wrapChatCompletionsAffinityEgress(observed, affinityEgressOptions(ctx)); - - if (!wantsStream) { - try { - const response = await collectChatCompletionsProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return internalChatCompletionsErrorResponse(502, toInternalDebugError(error)); - } - } - - forwardUpstreamHeaders(c, result.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, chatCompletionsSseFrames(frames, includeUsageChunk, state, ctx), { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`chat-completions stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); -}; - -// --- error rendering --- - -const internalChatCompletionsErrorPayload = (error: InternalDebugError) => ({ - error: { - type: error.type, - name: error.name, - message: error.message, - stack: error.stack, - cause: error.cause, - target_api: error.target_api, - }, -}); - -const internalChatCompletionsErrorResponse = (status: number, error: InternalDebugError): Response => Response.json(internalChatCompletionsErrorPayload(error), { status }); - -// --- frame observation --- - -const isChatCompletionsFailureFrame = (frame: ProtocolFrame) => frame.type === 'event' && chatCompletionsErrorPayloadMessage(frame.event) !== null; - -const isChatCompletionsTerminalFrame = (frame: ProtocolFrame) => frame.type === 'done' || isChatCompletionsFailureFrame(frame); - -const observeChatCompletionsFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - for await (const frame of frames) { - ctx.dump?.frame(frame); - const failed = isChatCompletionsFailureFrame(frame); - if (failed) state.failed = true; - if (isChatCompletionsTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isChatCompletionsTerminalFrame(frame)) return; - } - throw new Error(CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE); -}; - -const chatCompletionsSseFrames = async function* (frames: AsyncIterable>, includeUsageChunk: boolean, state: SourceStreamState, ctx: GatewayCtx) { - try { - for await (const frame of frames) { - const sse = chatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk }); - if (sse) yield sse; - } - } catch (error) { - state.failed = true; - const event = internalChatCompletionsErrorPayload(toInternalDebugError(error)) as unknown as ChatCompletionsStreamEvent; - ctx.dump?.frame(eventFrame(event)); - yield sseFrame(JSON.stringify(event), 'error'); - } -}; diff --git a/packages/gateway/src/data-plane/chat/chat-completions/serve.ts b/packages/gateway/src/data-plane/chat/chat-completions/serve.ts deleted file mode 100644 index ade1c6e2b6..0000000000 --- a/packages/gateway/src/data-plane/chat/chat-completions/serve.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; -import { chatCompletionsAttempt, chatCompletionsTarget } from './attempt.ts'; -import { renderChatCompletionsFailure } from './errors.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; - -export interface ChatCompletionsServeGenerateArgs { - readonly payload: ChatCompletionsPayload; - readonly ctx: ChatGatewayCtx; - readonly headers: Headers; -} - -export const chatCompletionsServe = { - generate: async (args: ChatCompletionsServeGenerateArgs): Promise>> => { - const { payload, ctx, headers } = args; - const affinity = await analyzeChatCompletionsAffinity(payload, ctx.affinity.codec); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model: payload.model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = enumerated.filter(c => chatCompletionsTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderChatCompletionsFailure(selection); - if (selection.candidates.length === 0) return renderChatCompletionsFailure(noViableCandidateFailure(sawModel, payload.model, failedUpstreams)); - - // Try each affinity-selected candidate in order. A successful attempt (SSE - // stream opened) is the final answer; an api-error or internal-error - // from one candidate falls through to the next so the gateway absorbs - // transient 5xx/429/network failures. When the list is exhausted, the - // most recent failure is forwarded verbatim so the client still sees - // real upstream telemetry rather than a synthetic envelope. Each attempt - // stamps its private payload clone with the candidate's canonical model id - // so aliases and prefixed ids resolve without mutating the caller payload. - return await iterateCandidates( - selection.candidates, - 'chatCompletionsServe.generate', - ctx, - 'chat', - async candidate => { - const result = await chatCompletionsAttempt.generate({ payload: selection.payloadFor(candidate), ctx, candidate, headers }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/gemini/attempt.ts b/packages/gateway/src/data-plane/chat/gemini/attempt.ts deleted file mode 100644 index 64b604a252..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/attempt.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { geminiStatusForHttpStatus } from './errors.ts'; -import { geminiCountTokensInterceptors, geminiInterceptors } from './interceptors/index.ts'; -import { stripUnsupportedPartFieldsFromPayload } from './interceptors/strip-unsupported-part-fields.ts'; -import { stripUnsupportedToolsFromPayload } from './interceptors/strip-unsupported-tools.ts'; -import { chatCompletionsAttempt } from '../chat-completions/attempt.ts'; -import { messagesAttempt } from '../messages/attempt.ts'; -import { responsesAttempt } from '../responses/attempt.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import { type ModelCandidate, plainResult, type ExecuteResult, type GeminiInvocation, type PlainResult } from '@floway-dev/provider'; -import { translateGeminiViaChatCompletions, translateGeminiViaMessages, translateGeminiViaResponses } from '@floway-dev/translate'; - -// Gemini has no native upstream target in the provider API; prefer Chat -// Completions, then Messages, then Responses for generate. countTokens has -// no translation path beyond native Messages count_tokens. -export const geminiGenerateTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); -export const geminiCountTokensTarget = chatTargetPicker(['messages']); - -export interface GeminiAttemptGenerateArgs { - readonly payload: GeminiPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export interface GeminiAttemptCountTokensArgs { - readonly payload: GeminiPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export const geminiAttempt = { - generate: async (args: GeminiAttemptGenerateArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = structuredClone(sourcePayload); - const headers = new Headers(sourceHeaders); - const targetApi = geminiGenerateTarget.pick(candidate.model.endpoints); - const invocation: GeminiInvocation = { payload, candidate, targetApi, headers }; - return await runInterceptors(invocation, ctx, geminiInterceptors, async () => { - // Gemini has no native upstream target today — every targetApi we - // pick is reached via translation. The dispatch threads each branch - // through `traverseTranslation` so each inner attempt owns its own - // interceptor chain and rewrite. - const transCtx = { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - }; - if (targetApi === 'messages') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiViaMessages(p, transCtx), - translated => messagesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }), - ); - } - if (targetApi === 'responses') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiViaResponses(p, transCtx), - translated => responsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - if (targetApi === 'chat-completions') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiViaChatCompletions(p, transCtx), - translated => chatCompletionsAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - throw new Error(`geminiAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, - - countTokens: async (args: GeminiAttemptCountTokensArgs): Promise => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = structuredClone(sourcePayload); - const headers = new Headers(sourceHeaders); - const targetApi = geminiCountTokensTarget.pick(candidate.model.endpoints); - const invocation: GeminiInvocation = { payload, candidate, targetApi, headers }; - return await runInterceptors(invocation, ctx, geminiCountTokensInterceptors, async () => { - // Gemini countTokens has no native upstream; translate to Messages and - // delegate to `messagesAttempt.countTokens`, then reshape the Messages - // count_tokens reply into the Gemini `{ totalTokens }` envelope. The - // shipped Gemini interceptors that mutate the payload pre-dispatch - // cannot run via the countTokens interceptor list — the post-`run()` - // ones inspect event streams the result type cannot carry — so the - // payload-mutators are applied inline here before translation; the - // attempt-owned payload clone keeps the caller's source intact. - const transCtx = { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - }; - const cleaned = invocation.payload; - stripUnsupportedPartFieldsFromPayload(cleaned); - stripUnsupportedToolsFromPayload(cleaned); - delete cleaned.safetySettings; - const trip = await translateGeminiViaMessages(cleaned, transCtx); - const { stream: _stream, ...target } = trip.target; - const messagesResult = await messagesAttempt.countTokens({ - payload: target, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }); - return reshapeMessagesCountAsGemini(messagesResult); - }); - }, -}; - -// Reshape the Messages count_tokens body into the Gemini `{ totalTokens }` -// envelope. The upstream body shape is provider-specific: Anthropic emits -// `{ input_tokens }`, Copilot's translated count emits `{ total_tokens }`; -// either is accepted. A missing or non-numeric figure is surfaced as a -// 502 Google-RPC error so the caller sees a typed Gemini failure rather -// than a passthrough of the upstream shape. -const reshapeMessagesCountAsGemini = (messagesResult: PlainResult): PlainResult => { - if (messagesResult.status !== 200) { - // Empty upstream bodies fall back to a fixed message so the Google-RPC envelope is never empty. - const text = new TextDecoder().decode(messagesResult.body); - return geminiErrorPlainResult(messagesResult.status, text || 'Upstream token counting request failed.', messagesResult.upstreamId); - } - let decoded: unknown; - try { decoded = JSON.parse(new TextDecoder().decode(messagesResult.body)); } catch {} - const upstreamTokenCounts = decoded && typeof decoded === 'object' - ? decoded as { input_tokens?: unknown; total_tokens?: unknown } - : {}; - const totalTokens = typeof upstreamTokenCounts.input_tokens === 'number' - ? upstreamTokenCounts.input_tokens - : typeof upstreamTokenCounts.total_tokens === 'number' - ? upstreamTokenCounts.total_tokens - : null; - if (totalTokens === null) { - return geminiInternalPlainResult(502, new Error('Invalid upstream token counting response.')); - } - return plainResult( - 200, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ totalTokens })), - messagesResult.upstreamId, - ); -}; - -const geminiErrorPlainResult = (status: number, message: string, upstream?: string): PlainResult => plainResult( - status, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ error: { code: status, message, status: geminiStatusForHttpStatus(status) } })), - upstream, -); - -const geminiInternalPlainResult = (status: number, error: Error): PlainResult => plainResult( - status, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ - error: { - code: status, - message: error.message, - status: geminiStatusForHttpStatus(status), - type: 'internal_error', - name: error.name, - stack: error.stack, - }, - })), -); diff --git a/packages/gateway/src/data-plane/chat/gemini/interceptors/index.ts b/packages/gateway/src/data-plane/chat/gemini/interceptors/index.ts deleted file mode 100644 index 4478357895..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/interceptors/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { stripSafetySettings } from './strip-safety-settings.ts'; -import { stripUnsupportedPartFields } from './strip-unsupported-part-fields.ts'; -import { stripUnsupportedTools } from './strip-unsupported-tools.ts'; -import { suppressThoughtParts } from './suppress-thought-parts.ts'; -import type { GeminiCountTokensInterceptor, GeminiInterceptor } from './types.ts'; - -// Unified Gemini interceptor list for `generate`. All four entries below are -// unconditional protocol-shape cleanups required because Gemini-shape requests -// cannot ride verbatim through other targets, plus the post-stream thought -// suppression that hides Gemini-native thought parts unless the caller opted -// in. There is no target-side companion list — Gemini has no native upstream -// in our provider API, so everything happens on the source side regardless of -// the chosen target. -export const geminiInterceptors: readonly GeminiInterceptor[] = [ - stripUnsupportedPartFields, - stripUnsupportedTools, - stripSafetySettings, - suppressThoughtParts, -]; - -// countTokens always translates Gemini → Messages and calls the Messages -// count_tokens upstream, which returns a raw `Response` rather than an event -// stream. The shipped Gemini interceptors all either mutate the payload pre- -// dispatch (acceptable) or wrap the post-`run()` event stream (incompatible -// with the count-tokens result shape). `geminiAttempt.countTokens` applies -// the payload-mutators inline before handing the translated payload to the -// Messages count_tokens path, so this list stays empty; the chain still runs -// so the count-tokens path keeps the same invocation envelope as generate. -export const geminiCountTokensInterceptors: readonly GeminiCountTokensInterceptor[] = []; diff --git a/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-safety-settings.ts b/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-safety-settings.ts deleted file mode 100644 index 2dec890067..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-safety-settings.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { GeminiInterceptor } from './types.ts'; - -/** - * Gemini safety controls are source-specific and have no matching control on - * every target path. Drop them so we don't pretend to enforce a policy we - * cannot honor end-to-end. - */ -export const stripSafetySettings: GeminiInterceptor = (ctx, _gatewayCtx, run) => { - delete ctx.payload.safetySettings; - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields.ts b/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields.ts deleted file mode 100644 index 528ca219a2..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-unsupported-part-fields.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { GeminiInterceptor } from './types.ts'; -import type { GeminiPayload, GeminiPart } from '@floway-dev/protocols/gemini'; - -/** - * Gemini file/code parts have no current equivalent in the upstream target - * graph. Drop them at source so every Gemini route target sees translatable - * parts. - */ -const stripPartFields = (parts: GeminiPart[]): GeminiPart[] => - parts.filter(part => { - delete part.fileData; - delete part.executableCode; - delete part.codeExecutionResult; - return Object.keys(part).length > 0; - }); - -export const stripUnsupportedPartFieldsFromPayload = (payload: GeminiPayload): void => { - payload.contents?.forEach(content => { - content.parts = stripPartFields(content.parts); - }); - if (payload.systemInstruction) { - payload.systemInstruction.parts = stripPartFields(payload.systemInstruction.parts); - } -}; - -export const stripUnsupportedPartFields: GeminiInterceptor = (ctx, _gatewayCtx, run) => { - stripUnsupportedPartFieldsFromPayload(ctx.payload); - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-unsupported-tools.ts b/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-unsupported-tools.ts deleted file mode 100644 index 48a1616d27..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/interceptors/strip-unsupported-tools.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { GeminiInterceptor } from './types.ts'; -import type { GeminiPayload, GeminiToolGroup } from '@floway-dev/protocols/gemini'; - -/** - * Only function declarations are currently translatable from Gemini tool - * groups. Strip the rest after target planning so target emitters never see - * unsupported tool capabilities. - */ -const stripToolCapabilities = (tool: GeminiToolGroup): void => { - delete tool.googleSearch; - delete tool.googleSearchRetrieval; - delete tool.codeExecution; - delete tool.computerUse; - delete tool.urlContext; - delete tool.fileSearch; - delete tool.mcpServers; - delete tool.googleMaps; -}; - -export const stripUnsupportedToolsFromPayload = (payload: GeminiPayload): void => { - if (!payload.tools) return; - - const tools = payload.tools.filter(tool => { - stripToolCapabilities(tool); - return tool.functionDeclarations && tool.functionDeclarations.length > 0; - }); - - if (tools.length === 0) { - delete payload.tools; - } else { - payload.tools = tools; - } -}; - -export const stripUnsupportedTools: GeminiInterceptor = (ctx, _gatewayCtx, run) => { - stripUnsupportedToolsFromPayload(ctx.payload); - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/gemini/interceptors/suppress-thought-parts.ts b/packages/gateway/src/data-plane/chat/gemini/interceptors/suppress-thought-parts.ts deleted file mode 100644 index fc99180aa7..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/interceptors/suppress-thought-parts.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { GeminiInterceptor } from './types.ts'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiStreamEvent } from '@floway-dev/protocols/gemini'; - -const hasEventPayload = (event: GeminiStreamEvent): boolean => { - if ('error' in event) return true; - return (event.candidates?.length ?? 0) > 0 || event.usageMetadata !== undefined || event.modelVersion !== undefined || event.responseId !== undefined; -}; - -const suppressThoughtPartsFromFrames = async function* (frames: AsyncIterable>): AsyncGenerator> { - for await (const frame of frames) { - if (frame.type !== 'event' || 'error' in frame.event) { - yield frame; - continue; - } - - const candidates = frame.event.candidates?.flatMap(candidate => { - const parts = candidate.content.parts.filter(part => part.thought !== true); - if (!parts.length && candidate.finishReason === undefined) return []; - - return [ - { - ...candidate, - content: { ...candidate.content, parts }, - }, - ]; - }); - - const event: GeminiStreamEvent = { - ...frame.event, - ...(candidates !== undefined ? { candidates } : {}), - }; - - if (hasEventPayload(event)) yield eventFrame(event); - } -}; - -/** - * Hide Gemini thought-summary parts unless the caller explicitly opted in via - * `generationConfig.thinkingConfig.includeThoughts === true`. - */ -export const suppressThoughtParts: GeminiInterceptor = async (ctx, _gatewayCtx, run) => { - const result = await run(); - if (result.type !== 'events' || ctx.payload.generationConfig?.thinkingConfig?.includeThoughts === true) { - return result; - } - - return { ...result, events: suppressThoughtPartsFromFrames(result.events) }; -}; diff --git a/packages/gateway/src/data-plane/chat/gemini/interceptors/types.ts b/packages/gateway/src/data-plane/chat/gemini/interceptors/types.ts deleted file mode 100644 index 612414fbac..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/interceptors/types.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import type { ExecuteResult, GeminiInvocation, PlainResult } from '@floway-dev/provider'; - -export type GeminiInterceptor = Interceptor< - GeminiInvocation, - GatewayCtx, - ExecuteResult> ->; - -// countTokens is a one-shot, non-streaming HTTP exchange — the terminal -// returns a `PlainResult` carrying the reshaped Gemini envelope, not an event -// stream. The interceptor chain still runs against a `GeminiInvocation` so -// payload-shaped reads stay symmetric with the generate path. Interceptors -// registered here MUST be pure header/payload mutators; post-`run()` result -// inspection is not portable to this result type. -export type GeminiCountTokensInterceptor = Interceptor< - GeminiInvocation, - GatewayCtx, - PlainResult ->; diff --git a/packages/gateway/src/data-plane/chat/gemini/serve.ts b/packages/gateway/src/data-plane/chat/gemini/serve.ts deleted file mode 100644 index 3549ae8e50..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/serve.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { analyzeGeminiAffinity } from './affinity/ingress.ts'; -import { geminiAttempt, geminiCountTokensTarget, geminiGenerateTarget } from './attempt.ts'; -import { renderGeminiFailure } from './errors.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import type { ExecuteResult, PlainResult } from '@floway-dev/provider'; - -export interface GeminiServeGenerateArgs { - readonly payload: GeminiPayload; - readonly ctx: ChatGatewayCtx; - // Per-request model id (Gemini carries it in the URL path, not the body), - // resolved by the HTTP entry and threaded through here so candidate - // enumeration and failure rendering all see the same value. - readonly model: string; - readonly headers: Headers; -} - -export interface GeminiServeCountTokensArgs { - readonly payload: GeminiPayload; - readonly ctx: ChatGatewayCtx; - readonly model: string; - readonly headers: Headers; -} - -export const geminiServe = { - generate: async (args: GeminiServeGenerateArgs): Promise>> => { - const { payload, ctx, model, headers } = args; - const affinity = await analyzeGeminiAffinity(payload, ctx.affinity.codec); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = enumerated.filter(c => geminiGenerateTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderGeminiFailure(selection, 'generate'); - if (selection.candidates.length === 0) return renderGeminiFailure(noViableCandidateFailure(sawModel, model, failedUpstreams), 'generate'); - - // Gemini carries the requested model in its URL, so affinity preparation - // owns each candidate payload while dispatch uses the candidate's canonical model. - return await iterateCandidates( - selection.candidates, - 'geminiServe.generate', - ctx, - 'chat', - async candidate => { - const result = await geminiAttempt.generate({ payload: selection.payloadFor(candidate), ctx, candidate, headers }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - }, - - countTokens: async (args: GeminiServeCountTokensArgs): Promise> | PlainResult> => { - const { payload, ctx, model, headers } = args; - const affinity = await analyzeGeminiAffinity(payload, ctx.affinity.codec); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = enumerated.filter(c => geminiCountTokensTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderGeminiFailure(selection, 'countTokens'); - if (selection.candidates.length === 0) return renderGeminiFailure(noViableCandidateFailure(sawModel, model, failedUpstreams), 'countTokens'); - - return await iterateCandidates( - selection.candidates, - 'geminiServe.countTokens', - ctx, - 'chat', - candidate => geminiAttempt.countTokens({ payload: selection.payloadFor(candidate), ctx, candidate, headers }), - ); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/messages/attempt.ts b/packages/gateway/src/data-plane/chat/messages/attempt.ts deleted file mode 100644 index f371379f7b..0000000000 --- a/packages/gateway/src/data-plane/chat/messages/attempt.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { messagesInterceptors, messagesCountTokensInterceptors } from './interceptors/index.ts'; -import type { MessagesInvocation } from './interceptors/types.ts'; -import { createMessagesBillableUsageReader } from './usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; -import { chatCompletionsAttempt } from '../chat-completions/attempt.ts'; -import { responsesAttempt } from '../responses/attempt.ts'; -import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerStreamResultToExecuteResult } from '../shared/provider-stream-result.ts'; -import { plainResultFromResponse } from '../shared/respond.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ModelCandidate, ExecuteResult, MessagesUpstreamCallOptions, PlainResult } from '@floway-dev/provider'; -import { providerModelOf } from '@floway-dev/provider'; -import { translateMessagesViaChatCompletions, translateMessagesViaResponses } from '@floway-dev/translate'; - -// `/v1/messages` generate prefers a native Messages target, then the -// translated Responses path, then the translated Chat Completions path. -export const messagesGenerateTarget = chatTargetPicker(['messages', 'responses', 'chat-completions']); - -// `count_tokens` has no translation path — only a native Messages target -// satisfies the operation. -export const messagesCountTokensTarget = chatTargetPicker(['messages']); - -export interface MessagesAttemptArgs { - readonly payload: MessagesPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; - readonly anthropicBeta: readonly string[]; -} - -const buildMessagesUpstreamCallOptions = ( - candidate: ModelCandidate, - ctx: ChatGatewayCtx, - headers: Headers, - anthropicBeta: readonly string[], -): MessagesUpstreamCallOptions => ({ - ...buildUpstreamCallOptions(candidate, ctx, headers), - anthropicBeta, -}); - -export const messagesAttempt = { - generate: async (args: MessagesAttemptArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders, anthropicBeta } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - headers.delete('anthropic-beta'); - const targetApi = messagesGenerateTarget.pick(candidate.model.endpoints); - const invocation: MessagesInvocation = { - payload, - candidate, - targetApi, - headers, - }; - return await runInterceptors(invocation, ctx, messagesInterceptors, async () => { - if (targetApi === 'messages') { - if (candidate.rules !== undefined) applyRulesToUpstreamMessages(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const providerResult = await candidate.provider.instance.callMessages( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildMessagesUpstreamCallOptions(candidate, ctx, invocation.headers, anthropicBeta), - ); - return await providerStreamResultToExecuteResult(providerResult, candidate, targetApi, ctx, createMessagesBillableUsageReader()); - } - if (targetApi === 'responses') { - return await traverseTranslation( - invocation.payload, - p => translateMessagesViaResponses(p, { model: candidate.model.id }), - translated => responsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - if (targetApi === 'chat-completions') { - return await traverseTranslation( - invocation.payload, - p => translateMessagesViaChatCompletions(p, { model: candidate.model.id }), - translated => chatCompletionsAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - throw new Error(`messagesAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, - - countTokens: async (args: MessagesAttemptArgs): Promise => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders, anthropicBeta } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - headers.delete('anthropic-beta'); - // `pick` here is contractually total — serve filtered with - // `messagesCountTokensTarget.canServe`, so a non-messages candidate is - // a contract breach. - const targetApi = messagesCountTokensTarget.pick(candidate.model.endpoints); - const invocation: MessagesInvocation = { - payload, - candidate, - targetApi, - headers, - }; - const response = await runInterceptors(invocation, ctx, messagesCountTokensInterceptors, async () => { - if (candidate.rules !== undefined) applyRulesToUpstreamMessages(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const { response } = await candidate.provider.instance.callMessagesCountTokens( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildMessagesUpstreamCallOptions(candidate, ctx, invocation.headers, anthropicBeta), - ); - return response; - }); - return await plainResultFromResponse(response, candidate.provider.upstreamId); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/messages/interceptors/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/messages/interceptors/apply-role-compatibility.ts deleted file mode 100644 index 38c808d338..0000000000 --- a/packages/gateway/src/data-plane/chat/messages/interceptors/apply-role-compatibility.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { MessagesPayloadInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withRoleCompatibilityApplied: MessagesPayloadInterceptor = (ctx, _gatewayCtx, run) => { - if (ctx.targetApi !== 'messages') return run(); - if (!providerModelOf(ctx.candidate).enabledFlags.has('rewrite-mid-conv-system-to-user')) return run(); - - ctx.payload = { - ...ctx.payload, - messages: ctx.payload.messages.map(message => - message.role === 'system' ? { role: 'user' as const, content: message.content } : message), - }; - - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice.ts deleted file mode 100644 index ea0c6401f2..0000000000 --- a/packages/gateway/src/data-plane/chat/messages/interceptors/disable-reasoning-on-forced-tool-choice.ts +++ /dev/null @@ -1,32 +0,0 @@ - -import type { MessagesPayloadInterceptor } from './types.ts'; -import type { MessagesPayload } from '@floway-dev/protocols/messages'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams where forced `tool_choice` and enabled -// thinking do not compose. Messages has a native `thinking: disabled` shape. -const hasForcedToolChoice = (payload: MessagesPayload): boolean => { - const type = payload.tool_choice?.type; - return type === 'tool' || type === 'any'; -}; - -const disableMessagesReasoning = (payload: MessagesPayload): MessagesPayload => { - // Strip only the reasoning subfield (`effort`) so structured-output - // `output_config.format` survives — forced tool choice composes fine with - // structured output on these upstreams, only with thinking does it not. - // If output_config becomes empty after the strip, omit it entirely. - const { output_config, ...rest } = payload; - const next: MessagesPayload = { ...rest, thinking: { type: 'disabled' as const } }; - if (output_config) { - const { effort: _effort, ...remaining } = output_config; - if (Object.keys(remaining).length > 0) next.output_config = remaining; - } - return next; -}; - -export const withReasoningDisabledOnForcedToolChoice: MessagesPayloadInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('disable-reasoning-on-forced-tool-choice')) return await run(); - if (!hasForcedToolChoice(ctx.payload)) return await run(); - ctx.payload = disableMessagesReasoning(ctx.payload); - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/messages/interceptors/index.ts b/packages/gateway/src/data-plane/chat/messages/interceptors/index.ts deleted file mode 100644 index 30d754d9b3..0000000000 --- a/packages/gateway/src/data-plane/chat/messages/interceptors/index.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { answerClaudeCodeProbe } from './answer-claude-code-probe.ts'; -import { withRoleCompatibilityApplied } from './apply-role-compatibility.ts'; -import { withReasoningDisabledOnForcedToolChoice } from './disable-reasoning-on-forced-tool-choice.ts'; -import { stripBillingAttribution } from './strip-billing-attribution.ts'; -import type { MessagesCountTokensInterceptor, MessagesInterceptor, MessagesPayloadInterceptor } from './types.ts'; -import { withMessagesWebSearchRequestPrepared, withMessagesWebSearchShim } from './web-search-shim.ts'; - -// Unified Messages generation chain. All entries are attached to every -// candidate; each interceptor decides whether to act — from the candidate's -// model flags and selected target, or, for the client-compatibility entry, -// from the inbound request itself. -// -// Translated requests re-enter the selected target protocol's chain. The role -// compatibility entry therefore acts only when Messages is the final target. -// -// - answerClaudeCodeProbe: registered ahead of everything else because it -// replaces the turn rather than shaping it — no payload transform below it -// can matter to a turn that never dials. Unconditional: whether a request -// is one of Claude Code's one-token probes is a property of the request, -// not of the candidate. -// - withMessagesWebSearchShim: registered next so its request preparation, -// replay rewrite, and intercept loop wrap the rest of the generation chain. -// Unconditional for translated targets (Responses / Chat Completions cannot -// carry Anthropic server tools); gated by `messages-web-search-shim` for -// native Messages targets. count_tokens runs the same request preparation -// without the stream-response wrapper. -// - stripBillingAttribution: gated by `strip-billing-attribution` (default -// on for copilot/azure/custom, off for claude-code). On candidates -// where it runs, it scrubs Claude Code's `x-anthropic-billing-header` / -// `cch=` markers out of the source-shape system prompt so prompt-cache -// hits survive across requests; on claude-code, the block is left intact -// because Anthropic uses it for plan-tier billing. -// - withReasoningDisabledOnForcedToolChoice: gated by -// `disable-reasoning-on-forced-tool-choice`. -// - withRoleCompatibilityApplied: Anthropic's top-level `payload.system` is -// the only first-position system slot, so the mid-conversation-system flag -// rewrites every inline system message to user after Messages is selected -// as the final target. -// -// The remaining three entries mutate only the request payload and are shared -// with count_tokens in the same order. Token counting therefore observes the -// same gateway-level billing-attribution, reasoning, and role shape as -// generation. -const messagesPayloadInterceptors: readonly MessagesPayloadInterceptor[] = [ - stripBillingAttribution, - withReasoningDisabledOnForcedToolChoice, - withRoleCompatibilityApplied, -]; - -export const messagesInterceptors: readonly MessagesInterceptor[] = [ - answerClaudeCodeProbe, - withMessagesWebSearchShim, - ...messagesPayloadInterceptors, -]; - -export const messagesCountTokensInterceptors: readonly MessagesCountTokensInterceptor[] = [ - withMessagesWebSearchRequestPrepared, - ...messagesPayloadInterceptors, -]; diff --git a/packages/gateway/src/data-plane/chat/messages/interceptors/strip-billing-attribution.ts b/packages/gateway/src/data-plane/chat/messages/interceptors/strip-billing-attribution.ts deleted file mode 100644 index f37b75fcae..0000000000 --- a/packages/gateway/src/data-plane/chat/messages/interceptors/strip-billing-attribution.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { MessagesPayloadInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -// Claude Code clients seed every Messages request with an -// `x-anthropic-billing-header: …` line carrying a per-turn `cch=` value. -// Two upstreams want opposite things from that block: -// -// - The Claude Code subscription endpoint (provider kind `claude-code`) -// reads the block to bill the request against the user's plan tier. If -// we strip it, the request silently falls off plan billing. -// - Every other upstream (copilot, azure, custom) treats the block as -// ordinary prompt text. The `cch=` hash flips per call, so the -// upstream's prompt-cache layer sees a "different" prompt every turn -// and never reuses its cache, even when the real conversation prefix -// hasn't changed. -// -// Gating is owned by the `strip-billing-attribution` flag in the provider -// flag catalog; defaults are wired there per kind. -const BILLING_HEADER_LINE_RE = /x-anthropic-billing-header[^\n]*/g; -const CCH_HASH_RE = /cch=[0-9a-f]{5,};?/gi; - -const stripText = (text: string): string => text.replace(BILLING_HEADER_LINE_RE, '').replace(CCH_HASH_RE, '').trim(); - -export const stripBillingAttribution: MessagesPayloadInterceptor = (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('strip-billing-attribution')) return run(); - - const { payload } = ctx; - - if (typeof payload.system === 'string') { - const system = stripText(payload.system); - const { system: _system, ...rest } = payload; - ctx.payload = system.length > 0 ? { ...rest, system } : rest; - } else if (Array.isArray(payload.system)) { - const system = payload.system - .map(block => ({ ...block, text: stripText(block.text) })) - .filter(block => block.text.length > 0); - const { system: _system, ...rest } = payload; - ctx.payload = system.length > 0 ? { ...rest, system } : rest; - } - - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/messages/respond.ts b/packages/gateway/src/data-plane/chat/messages/respond.ts deleted file mode 100644 index d5ad60f2ee..0000000000 --- a/packages/gateway/src/data-plane/chat/messages/respond.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapMessagesAffinityEgress } from './affinity/egress.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import { eventFrame, type ProtocolFrame, sseFrame } from '@floway-dev/protocols/common'; -import { messagesProtocolFrameToSSEFrame, MESSAGES_MISSING_TERMINAL_MESSAGE, collectMessagesProtocolEventsToResult } from '@floway-dev/protocols/messages'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import { type ExecuteResult, type PlainResult, type InternalDebugError, toInternalDebugError } from '@floway-dev/provider'; -import { apiErrorToResponse } from '@floway-dev/provider'; - -// Renders an upstream Messages result into the client HTTP/SSE response. An -// error-typed result is a pre-stream failure and always answers as HTTP; an -// events result drains to one JSON body (non-streaming) or is proxied frame by -// frame (streaming). -export const respondMessages = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - ctx: GatewayCtx, -): Promise => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return apiErrorToResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return internalMessagesErrorResponse(result.status, result.error); - } - - if (result.type === 'plain') { - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); - } - - const state = new SourceStreamState(); - const observed = observeMessagesFrames(result.events, state, ctx); - const frames = wrapMessagesAffinityEgress(observed, affinityEgressOptions(ctx)); - - if (!wantsStream) { - try { - const response = await collectMessagesProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return internalMessagesErrorResponse(502, toInternalDebugError(error)); - } - } - - forwardUpstreamHeaders(c, result.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, messagesSseFrames(frames, state, ctx), { - keepAlive: { frame: sseFrame(JSON.stringify({ type: 'ping' }), 'ping') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`messages stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); -}; - -const internalMessagesErrorPayload = (error: InternalDebugError) => ({ - type: 'error', - error: { - type: error.type, - name: error.name, - message: error.message, - stack: error.stack, - cause: error.cause, - target_api: error.target_api, - }, -}); - -const internalMessagesErrorResponse = (status: number, error: InternalDebugError): Response => Response.json(internalMessagesErrorPayload(error), { status }); - -const isMessagesTerminalFrame = (frame: ProtocolFrame) => frame.type === 'event' && (frame.event.type === 'message_stop' || frame.event.type === 'error'); - -const observeMessagesFrames = async function* ( - frames: AsyncIterable>, - state: SourceStreamState, - ctx: GatewayCtx, -) { - for await (const frame of frames) { - ctx.dump?.frame(frame); - const failed = frame.type === 'event' && frame.event.type === 'error'; - if (failed) state.failed = true; - if (isMessagesTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isMessagesTerminalFrame(frame)) return; - } - throw new Error(MESSAGES_MISSING_TERMINAL_MESSAGE); -}; - -const messagesSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - try { - for await (const frame of frames) { - const sse = messagesProtocolFrameToSSEFrame(frame); - if (sse) yield sse; - } - } catch (error) { - state.failed = true; - const event = internalMessagesErrorPayload(toInternalDebugError(error)) as unknown as MessagesStreamEvent; - ctx.dump?.frame(eventFrame(event)); - yield sseFrame(JSON.stringify(event), 'error'); - } -}; diff --git a/packages/gateway/src/data-plane/chat/messages/serve.ts b/packages/gateway/src/data-plane/chat/messages/serve.ts deleted file mode 100644 index c7390e735f..0000000000 --- a/packages/gateway/src/data-plane/chat/messages/serve.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { analyzeMessagesAffinity } from './affinity/ingress.ts'; -import { messagesAttempt, messagesGenerateTarget, messagesCountTokensTarget } from './attempt.ts'; -import { renderMessagesFailure } from './errors.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { parseAnthropicBetaHeader, type MessagesPayload, type MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ExecuteResult, PlainResult } from '@floway-dev/provider'; - -export interface MessagesServeGenerateArgs { - readonly payload: MessagesPayload; - readonly ctx: ChatGatewayCtx; - readonly headers: Headers; -} - -export interface MessagesServeCountTokensArgs { - readonly payload: MessagesPayload; - readonly ctx: ChatGatewayCtx; - readonly headers: Headers; -} - -export const messagesServe = { - generate: async (args: MessagesServeGenerateArgs): Promise>> => { - const { payload, ctx, headers } = args; - const anthropicBeta = parseAnthropicBetaHeader(headers.get('anthropic-beta')); - const affinity = await analyzeMessagesAffinity(payload, ctx.affinity.codec); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model: payload.model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = enumerated.filter(c => messagesGenerateTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderMessagesFailure(selection, 'generate'); - if (selection.candidates.length === 0) return renderMessagesFailure(noViableCandidateFailure(sawModel, payload.model, failedUpstreams), 'generate'); - - // Try each affinity-selected candidate in order. A successful attempt (SSE - // stream opened) is the final answer; an api-error or internal-error - // from one candidate falls through to the next so the gateway absorbs - // transient 5xx/429/network failures. When the list is exhausted, the - // most recent failure is forwarded verbatim. Each attempt stamps its - // private payload clone with the candidate's canonical model id. - return await iterateCandidates( - selection.candidates, - 'messagesServe.generate', - ctx, - 'chat', - async candidate => { - const result = await messagesAttempt.generate({ payload: selection.payloadFor(candidate), ctx, candidate, headers, anthropicBeta }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - }, - - countTokens: async (args: MessagesServeCountTokensArgs): Promise> | PlainResult> => { - const { payload, ctx, headers } = args; - const anthropicBeta = parseAnthropicBetaHeader(headers.get('anthropic-beta')); - const affinity = await analyzeMessagesAffinity(payload, ctx.affinity.codec); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model: payload.model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = enumerated.filter(c => messagesCountTokensTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderMessagesFailure(selection, 'countTokens'); - if (selection.candidates.length === 0) return renderMessagesFailure(noViableCandidateFailure(sawModel, payload.model, failedUpstreams), 'countTokens'); - - return await iterateCandidates( - selection.candidates, - 'messagesServe.countTokens', - ctx, - 'chat', - candidate => messagesAttempt.countTokens({ payload: selection.payloadFor(candidate), ctx, candidate, headers, anthropicBeta }), - ); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/responses/attempt.ts b/packages/gateway/src/data-plane/chat/responses/attempt.ts deleted file mode 100644 index b4979b8317..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/attempt.ts +++ /dev/null @@ -1,244 +0,0 @@ -import { klona } from 'klona/json'; - -import { responsesInterceptors } from './interceptors/index.ts'; -import type { ResponsesAttemptResult, ResponsesInvocation } from './interceptors/types.ts'; -import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; -import { syntheticEventsFromCompaction } from './items/output.ts'; -import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; -import { chatCompletionsAttempt } from '../chat-completions/attempt.ts'; -import { messagesAttempt } from '../messages/attempt.ts'; -import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; -import { createExternalImageLoader } from '../shared/external-image-loader.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerStreamResultToExecuteResult } from '../shared/provider-stream-result.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectResponsesProtocolEventsToResult, type CanonicalResponsesPayload, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { type ModelCandidate, eventResult, readUpstreamApiError, providerModelOf, type ChatTargetApi, type ExecuteResult, type ProviderResponsesResult, type ResponsesAction } from '@floway-dev/provider'; -import { translateResponsesViaChatCompletions, translateResponsesViaMessages } from '@floway-dev/translate'; - -// `/v1/responses` generate prefers the native Responses target, then the -// translated Messages path, then the translated Chat Completions path. The -// same picker covers compact: every Responses target is reachable via the -// shim, which pivots compact→generate inside the chain on non-responses -// targets. -export const responsesTarget = chatTargetPicker(['responses', 'messages', 'chat-completions']); - -interface ResponsesAttemptBaseArgs { - readonly action: ResponsesAction; - readonly payload: CanonicalResponsesPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -interface ResponsesSourceState { - readonly privatePayloads: ReadonlyMap; -} - -export type ResponsesAttemptInvokeArgs = ResponsesAttemptBaseArgs & { - readonly sourceState?: ResponsesSourceState; -}; -type ResponsesAttemptGenerateArgs = Omit; - -// Single entry point for both `action: 'generate'` and `action: 'compact'`. -// Envelope-drain branches on the caller's intent (`action` passed by value), -// not on `invocation.action`. Interceptors are free to mutate `ctx.action` -// to steer inner dispatch — and, by the project's interceptor convention, -// they do not restore on the way out — so post-chain `invocation.action` -// reflects whatever the last writer left it at. The shape of the result we -// hand back is the caller's contract; keying off the caller's value is the -// only place that contract lives. -// -// The module-boundary invariant `compact-shaped ⇒ targetApi='responses'` -// at dispatch time is enforced in two places, each at the layer that owns -// the corresponding piece of state: -// -// - `invocation.action === 'compact'` is caught inside `dispatchResponses`'s -// `case 'messages'` / `case 'chat-completions'` arms — action is a -// Responses-level metadata field that the translators never see. -// - A `compaction_trigger` (or any other compact-shaped) item in input is -// caught by the translator itself — the `responses-via-messages` and -// `responses-via-chat-completions` translators reject any input-item -// variant they do not handle, so a compaction_trigger that slipped past -// the shim surfaces as a translator-level error rather than a silent -// drop. -// -// Both safety nets fire pre-upstream-call, live inside the chain (not after -// the interceptor finally blocks), and stay independent of the shim's -// presence. -// -// Responses state and client affinity both belong to the native source edge, -// outside this candidate attempt. Native serve passes the already-hydrated -// candidate payload plus any private source state; translated inner Responses -// calls pass only their translated payload. Keeping this function free of -// affinity decoding, state hydration, and persistence -// prevents an inner Responses target from owning another source protocol's -// client-visible state. -export const responsesAttempt = { - invoke: async (args: ResponsesAttemptInvokeArgs): Promise => { - const { action, ctx, candidate, headers: sourceHeaders } = args; - const headers = new Headers(sourceHeaders); - const targetApi = responsesTarget.pick(candidate.model.endpoints); - const payload = { ...klona(args.payload), model: candidate.model.id }; - ctx.store.beginAttempt(args.sourceState?.privatePayloads ?? new Map()); - // Copilot compaction and Azure-native compaction both emit assistant - // messages whose content blocks have `type: 'input_text'`, then refuse - // the same items echoed back as input on the next turn. Normalising - // here, after native serve has expanded stored `item_reference` items, - // catches both the direct-echo and store-replay paths in one place. - const normalized: CanonicalResponsesPayload = { ...payload, input: normalizeAssistantInputText(payload.input) }; - - const invocation: ResponsesInvocation = { - payload: normalized, - action, - candidate, - targetApi, - headers, - }; - const chainResult = await runInterceptors(invocation, ctx, responsesInterceptors, async () => - await dispatchResponses(invocation, ctx)); - - if (chainResult.type !== 'events') return chainResult; - - if (action === 'compact') { - const upstreamCompacted = await collectResponsesProtocolEventsToResult(chainResult.events); - return { - type: 'result', - result: upstreamCompacted, - modelIdentity: chainResult.modelIdentity, - usage: tokenUsageFromBillableUsage((await chainResult.finalMetadata)?.billableUsage), - performance: chainResult.performance, - }; - } - return chainResult; - }, - - // Narrowing wrapper for cross-protocol translation callers - // (Messages/Gemini/ChatCompletions translating into Responses) and the - // native HTTP/WS generate entry — both always run in generate mode and - // want the ExecuteResult branch. The compact branch is a contract - // violation here; an interceptor that pivoted generate→compact would - // surface as a throw, not a silent shape mismatch. - generate: async (args: ResponsesAttemptGenerateArgs): Promise>> => { - const result = await responsesAttempt.invoke({ ...args, action: 'generate' }); - if (result.type === 'result') { - throw new Error('responsesAttempt.generate received a compact result; an interceptor pivoted generate→compact unexpectedly'); - } - return result; - }, -}; - -const dispatchResponses = async ( - invocation: ResponsesInvocation, - ctx: ChatGatewayCtx, -): Promise>> => { - const { candidate, targetApi } = invocation; - switch (targetApi) { - case 'responses': { - if (candidate.rules !== undefined) applyRulesToUpstreamResponses(invocation.payload, candidate.rules); - // Compact drops `stream` and `store` before hitting the wire: `store` is a - // gateway-only snapshot-persistence hint the upstream compact endpoint - // rejects, and `stream` is irrelevant on a non-streaming call. The generate - // branch leaves both fields on the body — every provider's streaming call - // forces stream=true anyway. - let body: Omit; - if (invocation.action === 'compact') { - const { model: _model, stream: _stream, store: _store, ...rest } = invocation.payload; - body = rest; - } else { - const { model: _model, ...rest } = invocation.payload; - body = rest; - } - const providerResult = await candidate.provider.instance.callResponses( - providerModelOf(candidate), - body, - invocation.action, - ctx.abortSignal, - buildUpstreamCallOptions(candidate, ctx, invocation.headers), - ); - return await providerResponsesResultToExecuteResult(providerResult, candidate, targetApi, ctx); - } - case 'messages': - if (invocation.action === 'compact') { - // The responses-compact-shim is structurally required on non-responses - // targets and pivots ctx.action to 'generate' before reaching here; - // landing inside this case with action='compact' means the shim - // disengaged or was wired out of the chain. A compaction_trigger in - // input is caught one layer down by the translator's - // unexpected-input-item guard. - throw new Error(`responsesAttempt: action='compact' reached dispatch on targetApi='messages' — the responses-compact-shim must engage and pivot the action`); - } - return await traverseTranslation( - invocation.payload, - p => translateResponsesViaMessages(p, { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - loadRemoteImage: createExternalImageLoader(ctx.abortSignal), - }), - translated => messagesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }), - ); - case 'chat-completions': - if (invocation.action === 'compact') { - throw new Error(`responsesAttempt: action='compact' reached dispatch on targetApi='chat-completions' — the responses-compact-shim must engage and pivot the action`); - } - return await traverseTranslation( - invocation.payload, - p => translateResponsesViaChatCompletions(p, { model: candidate.model.id }), - translated => chatCompletionsAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - default: { - const exhaustive: never = targetApi; - throw new Error(`unexpected targetApi '${exhaustive as string}'`); - } - } -}; - -// Lowers a `ProviderResponsesResult` into the chain's -// ExecuteResult> currency. The compact -// branch synthesizes SSE frames from the envelope so every downstream -// interceptor sees the same event-stream contract regardless of which action -// the provider executed. -const providerResponsesResultToExecuteResult = async ( - providerResult: ProviderResponsesResult, - candidate: ModelCandidate, - targetApi: ChatTargetApi, - ctx: ChatGatewayCtx, -): Promise>> => { - if (providerResult.action === 'generate') { - return await providerStreamResultToExecuteResult( - providerResult.ok - ? { ok: true, events: providerResult.events, modelKey: providerResult.modelKey, ...(providerResult.headers ? { headers: providerResult.headers } : {}) } - : { ok: false, response: providerResult.response, modelKey: providerResult.modelKey }, - candidate, - targetApi, - ctx, - billableUsageFromResponsesEvent, - ); - } - const context = upstreamPerformanceContext(ctx, candidate, 'chat'); - if (!providerResult.ok) { - return { ...(await readUpstreamApiError(providerResult.response, candidate.provider.upstreamId)), performance: context }; - } - // A native compaction is a turn the upstream ran and charged for, and its - // body states the counts. - const modelIdentity = telemetryModelIdentity(candidate, providerResult.modelKey); - const billableUsage = billableUsageFromResponsesResult(providerResult.result); - return eventResult( - syntheticEventsFromCompaction(providerResult.result), - modelIdentity, - { - performance: context, - ...(billableUsage === null ? {} : { finalMetadata: Promise.resolve({ modelIdentity, billableUsage }) }), - }, - ); -}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/apply-role-compatibility.ts deleted file mode 100644 index ec2721da1f..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/apply-role-compatibility.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { ResponsesInterceptor } from './types.ts'; -import type { ResponsesInputItem } from '@floway-dev/protocols/responses'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withRoleCompatibilityApplied: ResponsesInterceptor = (ctx, _gatewayCtx, run) => { - if (ctx.targetApi !== 'responses') return run(); - - const flags = providerModelOf(ctx.candidate).enabledFlags; - const rewriteSystemToDeveloper = flags.has('rewrite-system-to-developer'); - const rewriteDeveloperToSystem = flags.has('rewrite-developer-to-system'); - const rewriteMidConvSystemToUser = flags.has('rewrite-mid-conv-system-to-user'); - if (!rewriteSystemToDeveloper && !rewriteDeveloperToSystem && !rewriteMidConvSystemToUser) return run(); - - let crossedLeadingSystemRun = false; - ctx.payload = { - ...ctx.payload, - input: ctx.payload.input.map(item => { - let mapped: ResponsesInputItem = item; - if (mapped.type === 'message' && rewriteSystemToDeveloper && mapped.role === 'system') { - mapped = { ...mapped, role: 'developer' }; - } - if (mapped.type === 'message' && rewriteDeveloperToSystem && mapped.role === 'developer') { - mapped = { ...mapped, role: 'system' }; - } - const isSystemMessage = mapped.type === 'message' && mapped.role === 'system'; - if (!crossedLeadingSystemRun && !isSystemMessage) crossedLeadingSystemRun = true; - if (rewriteMidConvSystemToUser && crossedLeadingSystemRun && mapped.type === 'message' && mapped.role === 'system') { - mapped = { ...mapped, role: 'user' }; - } - return mapped; - }), - }; - - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice.ts deleted file mode 100644 index 66ad9fc5ef..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/disable-reasoning-on-forced-tool-choice.ts +++ /dev/null @@ -1,25 +0,0 @@ - -import type { ResponsesInterceptor } from './types.ts'; -import type { ResponsesPayload } from '@floway-dev/protocols/responses'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams where forced `tool_choice` and enabled -// reasoning do not compose. Sets the gateway's canonical "no reasoning" -// sentinel `reasoning: { effort: 'none' }` (also Responses API's documented -// disable value). Any active Vendor: * flag's last-running normalizer then -// translates that into the vendor's wire form. Sibling fields on the -// `reasoning` object (e.g. `summary`) are dropped — they have no meaning -// when reasoning is disabled. -const hasForcedToolChoice = (payload: ResponsesPayload): boolean => { - const toolChoice = payload.tool_choice; - if (toolChoice === undefined || toolChoice === null) return false; - if (typeof toolChoice === 'string') return toolChoice === 'required'; - return true; -}; - -export const withReasoningDisabledOnForcedToolChoice: ResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('disable-reasoning-on-forced-tool-choice')) return await run(); - if (!hasForcedToolChoice(ctx.payload)) return await run(); - ctx.payload = { ...ctx.payload, reasoning: { effort: 'none' } }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/index.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/index.ts deleted file mode 100644 index a2b5f02cf2..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/index.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { withRoleCompatibilityApplied } from './apply-role-compatibility.ts'; -import { withResponsesCompactShim } from './compact-shim.ts'; -import { withReasoningDisabledOnForcedToolChoice } from './disable-reasoning-on-forced-tool-choice.ts'; -import { withExclusiveCachedTokensNormalized } from './normalize-exclusive-cached-tokens.ts'; -import { withResponsesServerToolShim } from './server-tool-shim.ts'; -import { imageGenerationServerTool } from './server-tools/image-generation.ts'; -import { webSearchServerTool } from './server-tools/web-search.ts'; -import { withPromptCacheKeyStripped } from './strip-prompt-cache-key.ts'; -import type { ResponsesInterceptor } from './types.ts'; -import { withVendorDeepSeekResponsesNormalize } from './vendor-deepseek-normalize.ts'; -import { withVendorQwenResponsesNormalize } from './vendor-qwen-normalize.ts'; - -// Unified Responses interceptor list. All entries are attached to every -// candidate; each interceptor's body decides whether to act (flag-gated entries -// early-return on `providerModelOf(ctx.candidate).enabledFlags.has(flagId)`). -// -// Translated requests re-enter the selected target protocol's chain. The role -// compatibility entry therefore acts only when Responses is the final target, -// after pairwise translation has finished. -// -// Order matters: earlier entries wrap later ones. -// - withResponsesCompactShim: runs outermost so the action pivot -// ('compact' → 'generate' for the inner summarization turn) is visible -// to every downstream interceptor + the provider terminal. Also -// responsible for inbound expansion of prior shim-encoded compaction -// items so the upstream sees the summarized history. -// - withResponsesServerToolShim: wraps the multi-turn ReAct loop around -// the rest of the chain. -// - withReasoningDisabledOnForcedToolChoice: gated by -// `disable-reasoning-on-forced-tool-choice`. -// - withRoleCompatibilityApplied: applies role flags in the fixed order -// `system → developer → system → user`; later rewrites are authoritative -// when flags overlap, and the final step affects only mid-conversation system. -// - withPromptCacheKeyStripped: gated by `strip-prompt-cache-key`. Drops -// the top-level `prompt_cache_key` field for upstreams that reject it -// as an unknown argument (e.g. Azure DeepSeek). Runs before vendor -// normalizers so vendor-specific translation sees the already-stripped -// canonical payload. -// - withExclusiveCachedTokensNormalized: unconditional on a Responses -// target. Folds the cache buckets back into `input_tokens` whenever -// `total_tokens` witnesses that the upstream reports them alongside it, -// and consults `usage-exclusive-cached-tokens` for the responses whose -// totals witness nothing — so the flag is a declaration input rather than -// a gate. -// - withVendor*ResponsesNormalize: gated by `vendor-`. Registered after -// the role-compatibility entry so each gets the final say on the outbound wire -// body. -export const responsesInterceptors: readonly ResponsesInterceptor[] = [ - withResponsesCompactShim, - withResponsesServerToolShim([ - webSearchServerTool, - imageGenerationServerTool, - ]), - withReasoningDisabledOnForcedToolChoice, - withRoleCompatibilityApplied, - withPromptCacheKeyStripped, - withExclusiveCachedTokensNormalized, - withVendorDeepSeekResponsesNormalize, - withVendorQwenResponsesNormalize, -]; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens.ts deleted file mode 100644 index 6af05446ed..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/normalize-exclusive-cached-tokens.ts +++ /dev/null @@ -1,76 +0,0 @@ -// Responses-shape counterpart of the Chat Completions normalizer of the same -// name. The evidence for the two conventions, the decision rule, and the two -// contradictions that raise are documented once, at -// `../../chat-completions/interceptors/normalize-exclusive-cached-tokens.ts` -// and at `foldsExclusiveCacheTokens`. -// -// Responses carries usage on `event.response.usage` and names the buckets -// `input_tokens` / `input_tokens_details.{cached_tokens, cache_write_tokens}`. -// Every event that carries a response resource repeats the whole resource, so -// the rewrite applies to each of them rather than to a single terminal frame. - -import type { ResponsesInterceptor } from './types.ts'; -import { asJsonObject, type JsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; -import { foldsExclusiveCacheTokens } from '../../../shared/telemetry/usage.ts'; -import { eventFrame } from '@floway-dev/protocols/common'; -import type { ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { providerModelOf } from '@floway-dev/provider'; - -const rewriteInboundUsage = ( - event: ResponsesStreamEvent, - declaredExclusive: boolean, - identity: string, -): ResponsesStreamEvent => { - if (!('response' in event)) return event; - const response = asJsonObject(event.response); - const usage = asJsonObject(response?.usage); - if (!response || !usage) return event; - const inputTokens = readJsonNumber(usage.input_tokens); - const outputTokens = readJsonNumber(usage.output_tokens); - if (inputTokens == null || outputTokens == null) return event; - - const details = asJsonObject(usage.input_tokens_details); - const cacheRead = readJsonNumber(details?.cached_tokens) ?? 0; - const cacheWrite = readJsonNumber(details?.cache_write_tokens) ?? 0; - if (cacheRead === 0 && cacheWrite === 0) return event; - - const fold = foldsExclusiveCacheTokens(declaredExclusive, { - inputTokens, - outputTokens, - totalTokens: readJsonNumber(usage.total_tokens) ?? undefined, - cacheRead, - cacheWrite, - }, identity); - if (!fold) return event; - - const nextUsage: JsonObject = { ...usage, input_tokens: inputTokens + cacheRead + cacheWrite }; - const nextResponse: JsonObject = { ...response, usage: nextUsage }; - return { ...event, response: nextResponse } as unknown as ResponsesStreamEvent; -}; - -export const withExclusiveCachedTokensNormalized: ResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - // Runs only where the wire it speaks about is; see the Chat Completions - // counterpart. - if (ctx.targetApi !== 'responses') return await run(); - - const model = providerModelOf(ctx.candidate); - const declaredExclusive = model.enabledFlags.has('usage-exclusive-cached-tokens'); - const identity = `${ctx.candidate.provider.upstreamId}/${model.id}`; - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = rewriteInboundUsage(frame.event, declaredExclusive, identity); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/strip-prompt-cache-key.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/strip-prompt-cache-key.ts deleted file mode 100644 index 97dc36d20a..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/strip-prompt-cache-key.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { ResponsesInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams that reject `prompt_cache_key` as an unknown -// request argument (e.g. Azure DeepSeek). Drop the top-level field before the -// request reaches the terminal. OpenAI-native and truly OpenAI-compatible -// Responses upstreams accept it for prefix-cache attribution, so removal only -// happens under the flag. -export const withPromptCacheKeyStripped: ResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('strip-prompt-cache-key')) return await run(); - if (ctx.payload.prompt_cache_key === undefined) return await run(); - const { prompt_cache_key: _stripped, ...rest } = ctx.payload; - ctx.payload = rest; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/vendor-deepseek-normalize.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/vendor-deepseek-normalize.ts deleted file mode 100644 index 6efa2c2b80..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/vendor-deepseek-normalize.ts +++ /dev/null @@ -1,44 +0,0 @@ -// DeepSeek wire-dialect normalizer for the Responses protocol. Always- -// attached; flag-gated by `vendor-deepseek`. Runs last among interceptors -// so it has the final say on the outbound wire body. -// -// Outbound (request → upstream): -// -// - `reasoning.effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel (produced when a Messages source had `thinking: { type: -// 'disabled' }`, etc.). DeepSeek uses a top-level -// `thinking: { type: 'disabled' }` field instead. We strip the entire -// `reasoning` object and emit the DeepSeek form. -// -// Inbound: nothing today — the Responses-target dialect quirks that exist -// on Chat (assistant `reasoning_content` field, `prompt_cache_*_tokens` -// usage) have no Responses-shape equivalent that has surfaced. Add hooks -// here if vendor-specific Responses inbound rewrites become necessary. -// -// Reference: -// - https://api-docs.deepseek.com/zh-cn/guides/thinking_mode - -import type { ResponsesInterceptor } from './types.ts'; -import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; -import { providerModelOf } from '@floway-dev/provider'; - -interface DeepSeekDisableField { - thinking?: { type: 'disabled' }; -} - -type CanonicalResponsesPayloadWithDeepSeekDisable = Omit & DeepSeekDisableField; - -const stripCanonicalReasoningSentinel = (payload: CanonicalResponsesPayload): CanonicalResponsesPayload => { - if (payload.reasoning?.effort !== 'none') return payload; - const { reasoning: _stripped, ...rest } = payload; - const out: CanonicalResponsesPayloadWithDeepSeekDisable = { ...rest, thinking: { type: 'disabled' } }; - return out as CanonicalResponsesPayload; -}; - -export const withVendorDeepSeekResponsesNormalize: ResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-deepseek')) return await run(); - - ctx.payload = stripCanonicalReasoningSentinel(ctx.payload); - - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/responses/respond.ts b/packages/gateway/src/data-plane/chat/responses/respond.ts deleted file mode 100644 index 6a74c1c70c..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/respond.ts +++ /dev/null @@ -1,183 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapResponsesClientEgress } from './client-output.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import { doneFrame, eventFrame, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; -import { responsesProtocolFrameToSSEFrame, RESPONSES_MISSING_TERMINAL_MESSAGE, collectResponsesProtocolEventsToResult } from '@floway-dev/protocols/responses'; -import { isResponsesTerminalEvent, type CanonicalResponsesPayload, type ClientResponseResource, type ClientResponsesStreamEvent, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { type ExecuteResult, type PlainResult, type InternalDebugError, toInternalDebugError } from '@floway-dev/provider'; -import { apiErrorToResponse } from '@floway-dev/provider'; - -// Renders a Responses failure that never opened a stream. Separate entry -// because a request that fails before its payload parses has no payload to -// answer with, and the events path below requires one. -export const respondResponsesFailure = ( - result: Exclude>, { type: 'events' }> | PlainResult, - ctx: GatewayCtx, -): Response => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return apiErrorToResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return internalResponsesErrorResponse(result.status, result.error); - } - - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); -}; - -// Renders an upstream Responses result into the client HTTP/SSE response. An -// events result drains to one JSON body (non-streaming) or is proxied frame by -// frame (streaming); anything else is a pre-stream failure. The egress it runs -// is the stored-items membrane's, so the context it takes is the chat one that -// carries the store and the affinity membrane. -export const respondResponses = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - ctx: ChatGatewayCtx, - request: CanonicalResponsesPayload, -): Promise => { - if (result.type !== 'events') return respondResponsesFailure(result, ctx); - - const state = new SourceStreamState(); - const observed = observeResponsesFrames(result.events, state, ctx); - const frames = wrapResponsesClientEgress(observed, ctx, request); - - if (!wantsStream) { - try { - const response = await collectResponsesProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed || response.status === 'failed'); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return internalResponsesErrorResponse(502, toInternalDebugError(error)); - } - } - - forwardUpstreamHeaders(c, result.headers); - const response = streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, responsesSseFrames(frames, state, ctx), { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`responses stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); - - return response; -}; - -// --- error rendering --- - -const internalResponsesErrorResponse = (status: number, error: InternalDebugError): Response => - Response.json({ - error: { - type: error.type, - name: error.name, - message: error.message, - stack: error.stack, - cause: error.cause, - target_api: error.target_api, - }, - }, { status }); - -// The spec nests the `error` event's payload under `error`, and both official -// SDKs key their mid-stream throw on exactly that key; the same fields at the -// top level are yielded to them as an ordinary event instead. -// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L170-L177 -// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/core/streaming.ts#L69-L71 -// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/_streaming.py#L87-L98 -const internalResponsesStreamErrorEvent = (error: unknown): ClientResponsesStreamEvent => { - const debug = toInternalDebugError(error); - return { - type: 'error', - error: { - message: debug.message, - code: debug.type, - name: debug.name, - stack: debug.stack, - cause: debug.cause, - target_api: debug.target_api, - }, - } as unknown as ClientResponsesStreamEvent; -}; - -// --- frame observation --- - -const isResponsesTerminalFrame = (frame: ProtocolFrame) => frame.type === 'event' && isResponsesTerminalEvent(frame.event); - -const observeResponsesFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - for await (const frame of frames) { - ctx.dump?.frame(frame); - const failed = frame.type === 'event' && (frame.event.type === 'error' || frame.event.type === 'response.failed'); - if (failed) state.failed = true; - if (isResponsesTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isResponsesTerminalFrame(frame)) return; - } - throw new Error(RESPONSES_MISSING_TERMINAL_MESSAGE); -}; - -// "Any error incurred while streaming will be followed by a `response.failed` -// event." -// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L430 -const responsesFailedEvent = (resource: ClientResponseResource, error: unknown): ClientResponsesStreamEvent => { - const debug = toInternalDebugError(error); - return { - type: 'response.failed', - response: { ...resource, status: 'failed', error: { code: debug.type, message: debug.message } }, - } as ClientResponsesStreamEvent; -}; - -const responsesSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - let announced: ClientResponseResource | undefined; - try { - for await (const frame of frames) { - if (frame.type === 'event' && 'response' in frame.event) announced = frame.event.response; - yield responsesProtocolFrameToSSEFrame(frame); - } - // The SSE transport terminates on the literal `[DONE]` payload: - // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx?plain=1#L84 - yield responsesProtocolFrameToSSEFrame(doneFrame()); - } catch (error) { - state.failed = true; - const errorEvent = internalResponsesStreamErrorEvent(error); - ctx.dump?.frame(eventFrame(errorEvent)); - yield sseFrame(JSON.stringify(errorEvent), 'error'); - if (announced !== undefined) { - const failedFrame = eventFrame(responsesFailedEvent(announced, error)); - ctx.dump?.frame(failedFrame); - yield responsesProtocolFrameToSSEFrame(failedFrame); - } - } -}; diff --git a/packages/gateway/src/data-plane/chat/responses/serve-prep.ts b/packages/gateway/src/data-plane/chat/responses/serve-prep.ts index 361e3b4f48..ed153d2ea3 100644 --- a/packages/gateway/src/data-plane/chat/responses/serve-prep.ts +++ b/packages/gateway/src/data-plane/chat/responses/serve-prep.ts @@ -1,16 +1,5 @@ -import { analyzeResponsesAffinity } from './affinity/ingress.ts'; -import { responsesTarget } from './attempt.ts'; -import { renderResponsesFailure, type ResponsesServeFailure } from './errors.ts'; -import { hydrateResponsesPayload } from './items/hydrate.ts'; import type { StatefulResponsesStore } from './items/store.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import type { AffinityCandidateSelection } from '../shared/affinity/index.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure, tryCatchChatServeFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { CanonicalResponsesPayload, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import type { ModelCandidate, ExecuteResult } from '@floway-dev/provider'; +import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; // Thrown when a request names a `previous_response_id` that the store cannot // resolve. The HTTP/WS entry layer catches this and renders the OpenAI-shaped @@ -58,63 +47,3 @@ export const expandPreviousResponseId = async ( ], }; }; - -export type ResponsesServePlan = - | { readonly kind: 'failure'; readonly result: ExecuteResult> } - | { - readonly kind: 'ready'; - readonly affinitySelection: AffinityCandidateSelection; - readonly privatePayloads: ReadonlyMap; - readonly candidates: readonly ModelCandidate[]; - }; - -// Runs the native source preparation both `responsesServe.generate` and -// `responsesServe.compact` need before dispatching to `responsesAttempt`: -// expand any `previous_response_id`, load and hydrate stored items, prepare -// affinity, stage the user input, and return the selected candidate list. -// Returns a rendered failure result when no candidate is viable so the -// caller can surface it directly without re-deriving the model-error -// branch. The caller iterates the candidates — a successful attempt is the -// final answer, a per-candidate failure falls through to the next entry. -export const prepareResponsesServePlan = async (args: { - readonly payload: CanonicalResponsesPayload; - readonly ctx: ChatGatewayCtx; -}): Promise => { - const { payload, ctx } = args; - const store = ctx.store; - const prepared = await expandPreviousResponseId(payload, store); - const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model: prepared.model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = candidates.filter(c => responsesTarget.canServe(c.model.endpoints)); - await store.loadInputItems(prepared.input, payload.input); - let hydrated: ReturnType; - try { - hydrated = hydrateResponsesPayload(prepared, store); - } catch (error) { - const failure = tryCatchChatServeFailure(error); - if (failure === null) throw error; - return { kind: 'failure', result: renderResponsesFailure(failure) }; - } - const affinity = await analyzeResponsesAffinity(hydrated.payload, ctx.affinity.codec); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return { kind: 'failure', result: renderResponsesFailure(selection) }; - // Stage the user-supplied input from the original payload — not the - // expansion's `item_reference` prefix — so the next-turn snapshot picks - // up the new user items in addition to the prior snapshot history. - // Runs after the affinity walk so any `item_reference` in user-supplied - // input has its target row loaded. - await store.stageInputItems(payload.input); - - if (selection.candidates.length === 0) { - return { - kind: 'failure', - result: renderResponsesFailure(noViableCandidateFailure(sawModel, prepared.model, failedUpstreams)), - }; - } - return { kind: 'ready', affinitySelection: selection, privatePayloads: hydrated.privatePayloads, candidates: selection.candidates }; -}; diff --git a/packages/gateway/src/data-plane/chat/responses/serve.ts b/packages/gateway/src/data-plane/chat/responses/serve.ts deleted file mode 100644 index 705cba8347..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/serve.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { responsesAttempt } from './attempt.ts'; -import { responsesCreatedAt, wrapResponsesStatefulOutput } from './client-output.ts'; -import { completeResponsesCompaction } from './compaction-resource.ts'; -import type { ResponsesAttemptResult } from './interceptors/types.ts'; -import { syntheticEventsFromCompaction } from './items/output.ts'; -import { prepareResponsesServePlan } from './serve-prep.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectResponsesProtocolEventsToResult, type CanonicalResponsesPayload, type ClientResponsesCompaction, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import type { ExecuteResult } from '@floway-dev/provider'; - -interface ResponsesServeArgs { - readonly payload: CanonicalResponsesPayload; - readonly ctx: ChatGatewayCtx; - readonly headers: Headers; -} - -export const responsesServe = { - generate: async (args: ResponsesServeArgs): Promise>> => { - const { payload, ctx, headers } = args; - const plan = await prepareResponsesServePlan({ payload, ctx }); - if (plan.kind === 'failure') return plan.result; - // Iterate the affinity-selected candidates: success (SSE stream opened) is the - // final answer; per-candidate failures fall through so a transient - // 5xx/429/network does not become the request's verdict when another - // candidate can serve. The last failure surfaces verbatim on exhaustion. - // Each attempt stamps its private prepared-payload clone with the - // candidate's canonical model id. - const result = await iterateCandidates( - plan.candidates, - 'responsesServe.generate', - ctx, - 'chat', - async candidate => { - const result = await responsesAttempt.generate({ - payload: plan.affinitySelection.payloadFor(candidate), - sourceState: { - privatePayloads: plan.privatePayloads, - }, - ctx, - candidate, - headers, - }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - return result; - }, - - compact: async (args: ResponsesServeArgs): Promise> => { - const { payload, ctx, headers } = args; - // Compact accepts `previous_response_id` (the official endpoint documents - // it). When present serve-prep expands it the same way generate does so - // stored history is hydrated before candidate dispatch. - // - // For non-responses targets the responses-compact-shim picks up the - // request inside the interceptor chain, flips action='compact' to - // 'generate', runs a SUMMARIZATION_PROMPT turn through translation, and - // re-tags the result as compact on the way out. - const plan = await prepareResponsesServePlan({ payload, ctx }); - if (plan.kind === 'failure') return plan.result; - const result = await iterateCandidates( - plan.candidates, - 'responsesServe.compact', - ctx, - 'chat', - async candidate => { - const result = await responsesAttempt.invoke({ - payload: plan.affinitySelection.payloadFor(candidate), - sourceState: { - privatePayloads: plan.privatePayloads, - }, - action: 'compact', - ctx, - candidate, - headers, - }); - if (result.type === 'result') ctx.affinity.select(candidate); - return result; - }, - ); - if (result.type !== 'result') return result; - - const stored = wrapResponsesStatefulOutput(syntheticEventsFromCompaction(result.result), ctx); - const persisted = await collectResponsesProtocolEventsToResult(stored); - return { - ...result, - result: completeResponsesCompaction(persisted, responsesCreatedAt(ctx)), - }; - }, -}; diff --git a/packages/gateway/src/data-plane/chat/shared/upstream-models-error.ts b/packages/gateway/src/data-plane/chat/shared/upstream-models-error.ts deleted file mode 100644 index a4de64406e..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/upstream-models-error.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { ProviderModelsUnavailableError } from '@floway-dev/provider'; - -// A custom upstream's `/models` endpoint failing is the user's first signal -// that their upstream is misconfigured — relay the verbatim status/body so -// the caller sees the upstream's diagnostic, not a generic gateway envelope. -// Network-level failures (no httpResponse) return null; callers fall through -// to their own internal-error rendering for those because there is no -// upstream body to relay. -export const providerModelsUnavailableResponse = (error: unknown): Response | null => { - if (!(error instanceof ProviderModelsUnavailableError) || !error.httpResponse) return null; - const { status, headers, body } = error.httpResponse; - return new Response(body, { status, headers }); -}; diff --git a/packages/gateway/src/data-plane/embeddings/usage.ts b/packages/gateway/src/data-plane/embeddings/usage.ts deleted file mode 100644 index d3d3c7756d..0000000000 --- a/packages/gateway/src/data-plane/embeddings/usage.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { TokenUsage } from '../../repo/types.ts'; -import { tokenUsage } from '../shared/telemetry/usage.ts'; - -export const tokenUsageFromEmbeddingsBody = (body: unknown): TokenUsage | null => { - if (!body || typeof body !== 'object') return null; - const { usage } = body as { usage?: unknown }; - if (!usage || typeof usage !== 'object') return null; - const promptTokens = (usage as { prompt_tokens?: unknown }).prompt_tokens; - return typeof promptTokens === 'number' ? tokenUsage({ input: promptTokens }) : null; -}; diff --git a/packages/gateway/src/data-plane/shared/iterate-candidates.ts b/packages/gateway/src/data-plane/shared/iterate-candidates.ts deleted file mode 100644 index bab0f7fccc..0000000000 --- a/packages/gateway/src/data-plane/shared/iterate-candidates.ts +++ /dev/null @@ -1,78 +0,0 @@ -import type { GatewayCtx } from './gateway-ctx.ts'; -import { upstreamPerformanceContext } from './telemetry/attribution.ts'; -import type { ModelCandidate, PerformanceOperation } from '@floway-dev/provider'; - -// A serve-layer attempt result counts as success when: -// - The SSE event stream actually opened (`type: 'events'`). Mid-stream -// failure is the upstream's responsibility from there on; a fresh -// attempt on a different upstream does not start once the client has -// begun consuming events. -// - The non-streaming envelope landed: `PlainResult` with a 2xx status, -// or the Responses-compact `{type:'result'}` envelope. -// `api-error` and `internal-error` are failures: the serve loop falls -// through to the next candidate. 4xx is on the failure side — 429 -// (rate-limit) is the responsibility of the upstream that issued it, and -// the gateway's candidate ordering exists to absorb that kind of -// transient. Passthrough serves feed in an enlarged `plain` shape that -// carries the raw upstream Response plus per-attempt telemetry alongside -// the status; the success discriminant is unchanged. -type IterableAttemptResult = - | { readonly type: 'events' } - | { readonly type: 'result' } - | { readonly type: 'plain'; readonly status: number } - | { readonly type: 'api-error' } - | { readonly type: 'internal-error' }; - -const isAttemptSuccess = (result: IterableAttemptResult): boolean => { - switch (result.type) { - case 'events': - case 'result': - return true; - case 'plain': - return result.status >= 200 && result.status < 300; - case 'api-error': - case 'internal-error': - return false; - } -}; - -// Tries each narrowed candidate in order and returns the first success. A -// per-candidate *failure result* falls through so a transient 5xx/429 on -// one upstream rolls over to the next; a thrown error leaves the loop and -// surfaces to the caller, so a dial failure does not advance. When the -// list is exhausted the most recent failure is returned so callers can -// forward it verbatim and clients still see real upstream telemetry rather -// than a synthetic gateway envelope. Callers are contractually required to -// hand in a non-empty candidate list — the empty-candidate branch renders -// each caller's own protocol-shaped "no viable candidate" envelope at the -// serve site. -// -// Owns per-attempt AttemptState: clears the two timing slots and stamps -// `ctx.attempt.telemetry` with the current candidate's -// `PerformanceTelemetryContext` synchronously BEFORE handing control to -// `run`. That way a mid-attempt throw (interceptor bug, translation -// error, provider-layer JS exception not represented as a ChatServeFailure) -// still attributes the perf error row to the throwing candidate: the -// outer catch reads `ctx.attempt.telemetry` and feeds it into -// `recordFailedRequest`. Callsites don't need to duplicate this stamp. -export const iterateCandidates = async ( - candidates: readonly ModelCandidate[], - invocationLabel: string, - ctx: GatewayCtx, - operation: PerformanceOperation, - run: (candidate: ModelCandidate) => Promise, -): Promise => { - let lastFailure: T | undefined; - for (const candidate of candidates) { - ctx.attempt.upstreamCallStartedAt = null; - ctx.attempt.firstOutputTokenAt = null; - ctx.attempt.telemetry = upstreamPerformanceContext(ctx, candidate, operation); - const result = await run(candidate); - if (isAttemptSuccess(result)) return result; - lastFailure = result; - } - if (lastFailure === undefined) { - throw new Error(`invariant broken: ${invocationLabel} exhausted candidates with neither success nor failure`); - } - return lastFailure; -}; From 588f3fe8ea4daaeaf8b151f27d032096a775293a Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 10:57:42 +0800 Subject: [PATCH 049/156] fix(chat): settle a measurement like every other run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both count-token chains composed no settlement, so the stage that writes what a run billed was simply absent from them. The row they write is empty either way today — nothing here is billable — which is why nothing caught it. The ruling is about the reason rather than the outcome: an upstream that began charging for the operation would provide a non-empty set and nothing else would change. A chain with no settlement stage has nowhere for that to go, and writing "count_tokens is exempt" hard-codes today's commercial arrangement into the architecture. --- .../__tests__/data-plane/chat/messages/http_test.ts | 7 +++++-- .../gateway/src/data-plane/chat/gemini/count-tokens.ts | 6 ++++++ .../gateway/src/data-plane/chat/messages/count-tokens.ts | 6 ++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/http_test.ts index fca20261f9..55d1a73eb8 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages/http_test.ts @@ -238,10 +238,13 @@ test('POST /v1/messages/count_tokens proxies the upstream measurement body', asy assertEquals(body.input_tokens, 99); assertEquals(callMessagesCountTokens.mock.calls.length, 1); - // Measuring is not generating: the turn bills nothing and times nothing, so it leaves - // neither a usage row nor a performance sample behind. + // A measurement goes through settlement like every other run. It names no billed entity — + // nothing here is billable today — but the row is written, because "count_tokens is exempt + // from billing" would hard-code today's commercial arrangement into the architecture. An + // upstream that began charging would provide a non-empty set and nothing else would change. await flushBackground(); assertEquals(await repo.usage.listAll(), []); + // No latency to report either: nothing dialled an upstream on a path that measures. assertEquals(await repo.performance.listAll(), []); }); diff --git a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts index 8ff80bede0..9db3a81554 100644 --- a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts @@ -27,6 +27,7 @@ import { renderGeminiError } from './errors.ts'; import type { GeminiFacts } from './pipeline.ts'; import { asJsonObject, readJsonNumber } from '../../../shared/json-helpers.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import { @@ -186,6 +187,11 @@ export type GeminiCountTokensExit = G< export const geminiCountTokensPipeline = (payload: GeminiPayload): Pipeline => compose('geminiCountTokens', [ emitGeminiTokenCount, + // A measurement goes through settlement like every other run. It provides an empty + // billed set because nothing here is billable today, not because the operation is + // exempt — an upstream that began charging for it would provide a non-empty one and + // nothing else would change. + writeSettlement(handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini'])), resolveChatCandidates(narrowing(payload)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini']), diff --git a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts index 540bdd88fc..b7fac70a6a 100644 --- a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts @@ -31,6 +31,7 @@ import { prepareMessagesWebSearchShimRequest } from './interceptors/web-search-s import type { MessagesFacts } from './pipeline.ts'; import type { Failure } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; @@ -286,6 +287,11 @@ export type MessagesCountTokensExit = M< export const messagesCountTokensPipeline = (payload: MessagesPayload): Pipeline => compose('messagesCountTokens', [ emitMessagesTokenCount, + // A measurement goes through settlement like every other run. It provides an empty + // billed set because nothing here is billable today, not because the operation is + // exempt — an upstream that began charging for it would provide a non-empty one and + // nothing else would change. + writeSettlement(handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages'])), resolveChatCandidates(narrowing(payload)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages']), From da9f098b832383588263301189b179f51b57249f Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 11:00:01 +0800 Subject: [PATCH 050/156] docs(chat): say what assembly checks, and stop describing a state that passed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four comments described the branch mid-migration: the shared stages still said nothing picks a wire yet, and the Responses hydration still named an entry layer that catches a throw the chain now answers with. Two test comments claimed more than assembly delivers. `compose` refuses a wire's own rule placed below a handoff that consumed its key, and it walks one array — the same rule left above the fork is invisible to it. What keeps a wire's rule off a translated turn is that the rule lives in the wire's chain, and an interceptor array states the order of what sits above the fork. --- .../__tests__/data-plane/chat-interceptors_test.ts | 7 +++++-- .../data-plane/chat/translated-wires_test.ts | 7 ++++--- .../src/data-plane/chat/responses/serve-prep.ts | 11 +++++------ packages/gateway/src/data-plane/chat/stages.ts | 7 +++---- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index 012b40851e..a09bec040d 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -577,10 +577,13 @@ const usageEvent = (usage: Record): ProtocolFrame): Record => (frame as { event: { response: { usage: Record } } }).event.response.usage; -// Each array below is the order that family's chain runs, and running it is what says the -// order is a property of the array rather than of any one stage: assembly refuses +// Each array below is the order a family's chain runs these rules in, and running it is what +// says the order is a property of the array rather than of any one stage: assembly refuses // declarations that do not line up, and a rewrite that had to see another rewrite's output // only sees it from where it sits. +// +// It is not that family's *whole* chain. Rules a wire owns run in the wire's own chain, below +// the stage that picks it, so what an array states is the order of what sits above the fork. describe('a family\'s interceptor array, in the order its chain runs it', () => { it('assembles and runs the Messages array', async () => { const { down } = await runChain('response.chat.messages', [ diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index dab73cb7b7..9f513f0159 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -445,9 +445,10 @@ describe('a rule that speaks about one protocol-s wire', () => { }); // The other half, and this one assembly says rather than a run: the wire's own rules read - // the wire's own request key, so `compose` refuses to place them where a handoff has taken - // it. A chain cannot be arranged to run the Messages rule on a turn on its way out of - // Messages, whatever anyone writes. + // the wire's own request key, so `compose` refuses to place them *below* a handoff that has + // taken it. It is one array's arrangement and not a proof — `compose` walks one array, and + // the same rule left above the fork is invisible to it. What keeps a wire's rule off a + // translated turn is that the rule lives in the wire's own chain. it('cannot be placed in a chain whose payload a handoff consumed', () => { const leaving = handOff({ from: { request: 'request.chat.messages', response: 'response.chat.messages' }, diff --git a/packages/gateway/src/data-plane/chat/responses/serve-prep.ts b/packages/gateway/src/data-plane/chat/responses/serve-prep.ts index ed153d2ea3..4ffb09e3c9 100644 --- a/packages/gateway/src/data-plane/chat/responses/serve-prep.ts +++ b/packages/gateway/src/data-plane/chat/responses/serve-prep.ts @@ -2,11 +2,10 @@ import type { StatefulResponsesStore } from './items/store.ts'; import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; // Thrown when a request names a `previous_response_id` that the store cannot -// resolve. The HTTP/WS entry layer catches this and renders the OpenAI-shaped -// 400 body verbatim — clients (codex) compare it byte-for-byte against -// upstream OpenAI's `previous_response_not_found` envelope, so the rendering -// stays at the entry boundary instead of being folded into the generic -// ChatServeFailure renderer. +// resolve. The stage that hydrates catches this and answers with the +// OpenAI-shaped 400 body verbatim — clients (codex) compare it byte-for-byte +// against upstream OpenAI's `previous_response_not_found` envelope, so what +// they read is the upstream's own wording rather than this gateway's. // // Verbatim payload cross-verified from real upstream captures: // - https://github.com/cline/cline/issues/9399 @@ -27,7 +26,7 @@ export class PreviousResponseNotFoundError extends Error { // then drops `previous_response_id` from the payload (the snapshot id is a // gateway concept and never reaches the upstream wire). Native-entry only: // translated payloads coming in from another protocol's attempt never carry -// `previous_response_id`, so this prep runs in serve and not in attempt. +// `previous_response_id`, so this runs above the fork and never on a wire. export const expandPreviousResponseId = async ( payload: CanonicalResponsesPayload, store: StatefulResponsesStore, diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts index f88a289084..d1ab6a8562 100644 --- a/packages/gateway/src/data-plane/chat/stages.ts +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -7,10 +7,9 @@ // upstream that will see it — so it enters the record below the fork, where there is one // payload to speak of, rather than above it where there are as many as there are candidates. // -// When the translated wires land, which wire a candidate is dialled on will be decided per -// candidate too, by the chain's last stage handing into the target's. Failover re-runs the -// whole suffix including that stage, so the next candidate re-picks with no mechanism of its -// own. Every chain here dials its own protocol, so nothing picks yet. +// Which wire a candidate is dialled on is decided per candidate too, by the chain's last +// stage handing into the target protocol's. Failover re-runs the whole suffix including that +// stage, so the next candidate re-picks with no mechanism of its own. import type { AttemptSelector, GatewayFacts } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; From 89471cfd3618c42049a2bacb47c8a8bee8f6c4d4 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 11:22:52 +0800 Subject: [PATCH 051/156] fix(chat): restore the Chat Completions rules that never became stages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six of the deleted interceptors had no successor, and one of them was holding up billing. Nothing forced `stream_options.include_usage` any more, so an OpenAI-compatible upstream sent no usage chunk, the meter's reading stayed undefined and the attempt was billed an entity with no quantities — every Chat Completions turn, and every Messages, Responses or Gemini turn that reached an upstream over this wire. All six belong to the wire rather than to the source chain: each speaks about one upstream's Chat Completions endpoint, so a turn that leaves for another protocol must not carry them and a turn that arrives from one must. They keep the order the onion ran them in, which is the same order in both directions — the usage chunk is asked for above everything, and coming back the vendor dialects have the first say, so the cache-bucket fold reads cache fields under OpenAI's names and the carrier split reads usage the fold has already settled. What the interceptor form said with `ctx.targetApi !== 'chat-completions'` is now said by position. The cache-bucket fold is one rule over a protocol's own field names, which the Responses stage of the same name reads too, and the two stream helpers every response-direction rule needed are written once beside it. Each rule fails a test of its own when reverted and is caught when dropped from the wire, and a streaming turn through a routed endpoint is checked to bill non-zero against an upstream that reports usage only when it is asked to. --- .../data-plane/chat-interceptors_test.ts | 492 +++++++++++++- .../chat/chatcompletions-pipeline_test.ts | 127 +++- .../data-plane/chat/translated-wires_test.ts | 39 ++ .../chat/chat-completions/pipeline.ts | 29 +- .../src/data-plane/chat/interceptors.ts | 616 ++++++++++++++++-- 5 files changed, 1244 insertions(+), 59 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index a09bec040d..13ede213ae 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -14,7 +14,10 @@ import { disableReasoningOnForcedToolChoiceForChatCompletions, disableReasoningOnForcedToolChoiceForMessages, disableReasoningOnForcedToolChoiceForResponses, + includeUsageStreamOptionsForChatCompletions, + normalizeExclusiveCachedTokensForChatCompletions, normalizeExclusiveCachedTokensForResponses, + normalizeUsageForChatCompletions, stripBillingAttributionFromMessages, stripPromptCacheKeyForChatCompletions, stripPromptCacheKeyForResponses, @@ -22,13 +25,16 @@ import { stripUnsupportedPartFieldsFromGemini, stripUnsupportedToolsFromGemini, suppressThoughtPartsFromGemini, + vendorDeepSeekNormalizeForChatCompletions, vendorDeepSeekNormalizeForResponses, + vendorKimiNormalizeForChatCompletions, + vendorQwenNormalizeForChatCompletions, vendorQwenNormalizeForResponses, } from '../../src/data-plane/chat/interceptors.ts'; import type { AttemptSelector } from '../../src/data-plane/pipeline/facts.ts'; import { compose, defineStage, move, run } from '@floway-dev/pipeline'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; import type { MessagesPayload } from '@floway-dev/protocols/messages'; import type { ResponsesInputItem, ResponsesPayload, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; @@ -210,6 +216,438 @@ describe('the Chat Completions interceptors, as stages', () => { }); expect(down['request.chat.chatCompletions']).toBe(original); }); + + // The usage chunk is what every stream here is billed from, so what the client asked for and + // what the upstream is asked for differ — and this is the rule that makes them differ. + it('asks the upstream for the usage chunk when the client sent no stream options', async () => { + const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([]), + }); + expect((down['request.chat.chatCompletions'] as ChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); + }); + + it('asks for it over a client that asked for it to be left out', async () => { + const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([], { stream_options: { include_usage: false } }), + }); + expect((down['request.chat.chatCompletions'] as ChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); + }); + + // Everything else on `stream_options` is the client's own. An upstream that reads a key our + // typed surface does not name still gets it. + it('keeps the client-s other stream options while it forces the one it wants', async () => { + const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([], { + stream_options: { extra: 'keep-me', include_usage: false } as unknown as ChatCompletionsPayload['stream_options'], + }), + }); + expect((down['request.chat.chatCompletions'] as ChatCompletionsPayload).stream_options) + .toEqual({ extra: 'keep-me', include_usage: true }); + }); + + it('leaves a payload that already asked for it alone, by identity', async () => { + const original = chatCompletionsPayload([], { stream_options: { include_usage: true } }); + const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': original, + }); + expect(down['request.chat.chatCompletions']).toBe(original); + }); + + // The spec puts the final usage on a carrier chunk of its own; an upstream that hangs it off + // the last delta is split in two so that everything downstream reads the one shape. + it('moves usage off the chunk that carried the last delta and onto a carrier of its own', async () => { + const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.chatCompletions', {}, + streamOf([eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 }, + } as unknown as ChatCompletionsStreamEvent)])); + const frames = await drain(up['response.chat.chatCompletions']); + + expect(frames).toHaveLength(2); + const delta = chatCompletionsEventOf(frames[0]!); + expect(delta.choices).toEqual([{ index: 0, delta: {}, finish_reason: 'stop' }]); + expect('usage' in delta).toBe(false); + const carrier = chatCompletionsEventOf(frames[1]!); + // The carrier is the same chunk otherwise: an id and a model a client can correlate. + expect(carrier).toMatchObject({ id: 'c1', model: 'm', choices: [] }); + expect(carrier.usage).toEqual({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 }); + }); + + it('hands a chunk that is already a carrier, and one with no usage at all, back by identity', async () => { + const carrier = eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], + usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_tokens_details: { cached_tokens: 60 } }, + } as unknown as ChatCompletionsStreamEvent); + const content = eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', + choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], + } as unknown as ChatCompletionsStreamEvent); + const done = doneFrame(); + + const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.chatCompletions', {}, + streamOf([carrier, content, done])); + const frames = await drain(up['response.chat.chatCompletions']); + + expect(frames).toEqual([carrier, content, done]); + }); + + // Naming a vendor's cache fields is the vendor stage's job, and it has already run by the + // time the answer reaches this one. A chunk still carrying them is one no vendor claimed. + it('leaves a vendor-s own cache fields where it found them', async () => { + const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.chatCompletions', {}, + streamOf([chatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 25, + })])); + const frames = await drain(up['response.chat.chatCompletions']); + + expect(chatCompletionsUsageOf(frames[0]!)).toEqual({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 25, + }); + }); + + // Verbatim from a Charm Hyper kimi-k3 turn: 479 + 13312 + 373 = 14164, so `prompt_tokens` + // counts only what the cache did not serve, and `total_tokens` witnesses it whatever anyone + // flagged. + it('folds the cache buckets into the input total when the totals witness it', async () => { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + 'route.attempt': attemptWith(), + }, streamOf([chatCompletionsUsageChunk({ + prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_tokens_details: { cached_tokens: 13312 }, + })])); + const frames = await drain(up['response.chat.chatCompletions']); + + expect(chatCompletionsUsageOf(frames[0]!)).toMatchObject({ + prompt_tokens: 13791, + // The total already counted the real input, and now agrees with it. + total_tokens: 14164, + prompt_tokens_details: { cached_tokens: 13312 }, + }); + }); + + // The flag settles the responses whose totals witness nothing, which makes it a declaration + // input rather than the gate — and it arrives on the attempt this stage read on the way + // down, so this is also what proves the closure carries it to the way back. + it('folds on the flag alone when the chunk states no total, cache writes included', async () => { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), + }, streamOf([chatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 10, prompt_tokens_details: { cached_tokens: 120, cache_creation_input_tokens: 80 }, + })])); + const frames = await drain(up['response.chat.chatCompletions']); + + expect(chatCompletionsUsageOf(frames[0]!)).toMatchObject({ prompt_tokens: 300 }); + }); + + it('hands an inclusive chunk, and one with no cache counts, back by identity', async () => { + const inclusive = chatCompletionsUsageChunk({ + prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_tokens_details: { cached_tokens: 400 }, + }); + const uncached = chatCompletionsUsageChunk({ prompt_tokens: 40, completion_tokens: 1, total_tokens: 41 }); + + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), + }, streamOf([uncached])); + expect(await drain(up['response.chat.chatCompletions'])).toEqual([uncached]); + + const { up: unflagged } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + 'route.attempt': attemptWith(), + }, streamOf([inclusive])); + expect(await drain(unflagged['response.chat.chatCompletions'])).toEqual([inclusive]); + }); + + // Both contradictions name the flag and the upstream, because the remedy is a setting an + // operator makes on that upstream. Neither may pass silently: one over-charges the input by + // the whole cached prefix, the other underflows it. + it('raises when the flag claims exclusive and the totals say inclusive', async () => { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), + }, streamOf([chatCompletionsUsageChunk({ + prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_tokens_details: { cached_tokens: 400 }, + })])); + + await expect(drain(up['response.chat.chatCompletions'])) + .rejects.toThrow(/usage-exclusive-cached-tokens is enabled/); + }); + + it('raises naming the flag when the cache counts underflow with no verdict', async () => { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + 'route.attempt': attemptWith(), + }, streamOf([chatCompletionsUsageChunk({ + prompt_tokens: 479, completion_tokens: 373, prompt_tokens_details: { cached_tokens: 13312 }, + })])); + + await expect(drain(up['response.chat.chatCompletions'])) + .rejects.toThrow(/enable usage-exclusive-cached-tokens/); + }); + + it('leaves an answer that is not a stream alone, by identity', async () => { + const answer = { kind: 'value' as const, body: null }; + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + 'route.attempt': attemptWith(), + }, answer); + expect(up['response.chat.chatCompletions']).toBe(answer); + }); +}); + +/** A `choices: []` carrier holding nothing but the counts under test — the shape the spec puts + * a stream's final usage on. */ +const chatCompletionsUsageChunk = (usage: Record): ProtocolFrame => + eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage, + } as unknown as ChatCompletionsStreamEvent); + +const chatCompletionsEventOf = (frame: ProtocolFrame): Record => + (frame as unknown as { event: Record }).event; + +const chatCompletionsUsageOf = (frame: ProtocolFrame): Record => + chatCompletionsEventOf(frame).usage as Record; + +const deepSeekDelta = (delta: Record): ProtocolFrame => + eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', + choices: [{ index: 0, delta, finish_reason: null }], + } as unknown as ChatCompletionsStreamEvent); + +const deltaOf = (frame: ProtocolFrame): Record => + (chatCompletionsEventOf(frame).choices as { delta: Record }[])[0]!.delta; + +/** An assistant turn as the gateway holds it: the canonical reasoning fields, and a tool call + * the replay of a multi-turn loop has to keep. */ +const assistantReplay = (reasoning: Record): ChatCompletionsPayload['messages'] => [ + { role: 'user', content: 'first turn' }, + { + role: 'assistant', content: null, ...reasoning, + tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{}' } }], + }, + { role: 'tool', tool_call_id: 'call_1', content: 'result' }, +] as unknown as ChatCompletionsPayload['messages']; + +const assistantOf = (payload: unknown): Record => + (payload as ChatCompletionsPayload).messages[1] as unknown as Record; + +describe('the Chat Completions vendor dialects, as stages', () => { + it('projects the canonical assistant reasoning onto the one field DeepSeek reads', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload(assistantReplay({ + reasoning_text: 'let me check the docs', + reasoning_opaque: 'opaque-blob', + reasoning_items: [{ type: 'reasoning', summary: [] }], + })), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + const assistant = assistantOf(down['request.chat.chatCompletions']); + + expect(assistant.reasoning_content).toBe('let me check the docs'); + // The other three are not DeepSeek's, and `reasoning_opaque` in particular is the + // canonical cross-turn signature it does not accept. + expect('reasoning_text' in assistant).toBe(false); + expect('reasoning_opaque' in assistant).toBe(false); + expect('reasoning_items' in assistant).toBe(false); + expect(assistant.tool_calls).toHaveLength(1); + }); + + it('writes that field from the summary items when the scalar is the thing that is missing', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload(assistantReplay({ + reasoning_items: [{ + type: 'reasoning', id: 'rs_1', + summary: [{ type: 'summary_text', text: 'step one. ' }, { type: 'summary_text', text: 'step two.' }], + }], + })), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + + expect(assistantOf(down['request.chat.chatCompletions']).reasoning_content).toBe('step one. step two.'); + }); + + it('strips the fields it cannot project even when there is nothing to project', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([ + { role: 'user', content: 'first turn' }, + { role: 'assistant', content: 'answer', reasoning_items: [{ type: 'reasoning' }], reasoning_opaque: 'opaque-chain' }, + ] as unknown as ChatCompletionsPayload['messages']), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + const assistant = assistantOf(down['request.chat.chatCompletions']); + + expect('reasoning_content' in assistant).toBe(false); + expect('reasoning_items' in assistant).toBe(false); + expect('reasoning_opaque' in assistant).toBe(false); + expect(assistant.content).toBe('answer'); + }); + + it('puts the reasoning sentinel into the shape DeepSeek reads', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([], { reasoning_effort: 'none' }), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + const after = down['request.chat.chatCompletions'] as Record; + + expect('reasoning_effort' in after).toBe(false); + expect(after.thinking).toEqual({ type: 'disabled' }); + }); + + it('puts the reasoning sentinel into the shape Qwen reads', async () => { + const { down } = await runStage(vendorQwenNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([], { reasoning_effort: 'none' }), + 'route.attempt': attemptWith('vendor-qwen'), + }); + const after = down['request.chat.chatCompletions'] as Record; + + expect('reasoning_effort' in after).toBe(false); + expect(after.enable_thinking).toBe(false); + }); + + // Only the sentinel means "no reasoning". A real effort is the client asking for reasoning, + // and DeepSeek and Qwen both take it in the canonical field. + it('leaves a payload that asked for real reasoning alone, by identity', async () => { + const original = chatCompletionsPayload([], { reasoning_effort: 'high' }); + for (const stage of [vendorDeepSeekNormalizeForChatCompletions, vendorQwenNormalizeForChatCompletions]) { + const { down } = await runStage(stage, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': original, + 'route.attempt': attemptWith('vendor-deepseek', 'vendor-qwen'), + }); + expect(down['request.chat.chatCompletions']).toBe(original); + } + }); + + // DeepSeek's structured output takes `json_object` and nothing else, so the schema goes + // rather than the request being refused for carrying it. + it('downgrades a JSON schema to the JSON object DeepSeek supports', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([], { + response_format: { + type: 'json_schema', + json_schema: { name: 'messages_response', strict: true, schema: { type: 'object' } }, + }, + }), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + + expect((down['request.chat.chatCompletions'] as ChatCompletionsPayload).response_format).toEqual({ type: 'json_object' }); + }); + + it('leaves a payload already asking for a JSON object alone, by identity', async () => { + const original = chatCompletionsPayload([], { response_format: { type: 'json_object' } }); + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': original, + 'route.attempt': attemptWith('vendor-deepseek'), + }); + expect(down['request.chat.chatCompletions']).toBe(original); + }); + + it('renames DeepSeek-s reasoning deltas to the canonical field', async () => { + const { up } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([]), + 'route.attempt': attemptWith('vendor-deepseek'), + }, streamOf([ + deepSeekDelta({ role: 'assistant' }), + deepSeekDelta({ reasoning_content: 'thinking...' }), + deepSeekDelta({ content: 'answer' }), + doneFrame(), + ])); + const frames = await drain(up['response.chat.chatCompletions']); + + expect(deltaOf(frames[1]!).reasoning_text).toBe('thinking...'); + expect('reasoning_content' in deltaOf(frames[1]!)).toBe(false); + // The deltas either side of it are not this rule's business. + expect(deltaOf(frames[0]!)).toEqual({ role: 'assistant' }); + expect(deltaOf(frames[2]!)).toEqual({ content: 'answer' }); + }); + + it('rewrites DeepSeek-s cache-hit count into the field the rules above it read', async () => { + const { up } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([]), + 'route.attempt': attemptWith('vendor-deepseek'), + }, streamOf([chatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, + prompt_cache_hit_tokens: 70, prompt_cache_miss_tokens: 30, + prompt_tokens_details: { upstream_metric: 'preserved' }, + })])); + const frames = await drain(up['response.chat.chatCompletions']); + const usage = chatCompletionsUsageOf(frames[0]!); + + expect(usage.prompt_tokens).toBe(100); + expect(usage.prompt_tokens_details).toEqual({ upstream_metric: 'preserved', cached_tokens: 70 }); + // The miss count is what is left of the input, which `prompt_tokens` already says. + expect('prompt_cache_hit_tokens' in usage).toBe(false); + expect('prompt_cache_miss_tokens' in usage).toBe(false); + }); + + it('rewrites Kimi-s flat cached count into that same field', async () => { + const { up } = await runStage(vendorKimiNormalizeForChatCompletions, 'response.chat.chatCompletions', { + 'route.attempt': attemptWith('vendor-kimi'), + }, streamOf([chatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, cached_tokens: 50, + prompt_tokens_details: { upstream_metric: 'preserved' }, + })])); + const frames = await drain(up['response.chat.chatCompletions']); + const usage = chatCompletionsUsageOf(frames[0]!); + + expect(usage.prompt_tokens_details).toEqual({ upstream_metric: 'preserved', cached_tokens: 50 }); + expect('cached_tokens' in usage).toBe(false); + }); + + // An upstream that answers with an array where the details object should be is not carrying + // details, so what goes on the rewritten block is the one count that was read — not the + // array's own indices spread across it. + it('replaces a details field that is not an object rather than spreading it', async () => { + for (const [stage, flag, vendorField] of [ + [vendorDeepSeekNormalizeForChatCompletions, 'vendor-deepseek', 'prompt_cache_hit_tokens'], + [vendorKimiNormalizeForChatCompletions, 'vendor-kimi', 'cached_tokens'], + ] as const) { + const { up } = await runStage(stage, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([]), + 'route.attempt': attemptWith(flag), + }, streamOf([chatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, + [vendorField]: 70, prompt_tokens_details: [{ cached_tokens: 1 }], + })])); + const frames = await drain(up['response.chat.chatCompletions']); + + expect(chatCompletionsUsageOf(frames[0]!).prompt_tokens_details).toEqual({ cached_tokens: 70 }); + } + }); + + // A flag is data on an upstream model. A candidate that does not carry one is an upstream + // whose wire is already canonical, and nothing here may touch it. + it('stands down on a candidate that carries none of the vendor flags', async () => { + const payload = chatCompletionsPayload([], { reasoning_effort: 'none' }); + const usage = chatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 50, + }); + for (const stage of [ + vendorDeepSeekNormalizeForChatCompletions, + vendorQwenNormalizeForChatCompletions, + vendorKimiNormalizeForChatCompletions, + ]) { + const { down, up } = await runStage(stage, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': payload, + 'route.attempt': attemptWith(), + }, streamOf([usage])); + + expect(down['request.chat.chatCompletions']).toBe(payload); + expect(await drain(up['response.chat.chatCompletions'])).toEqual([usage]); + } + }); + + // A stream that breaks is the upstream's failure and belongs to whoever is reading it; a + // rewrite that swallowed it would present a broken turn as a whole one. + it('lets an upstream stream failure through as it was thrown', async () => { + const failure = new Error('upstream stream failed'); + for (const [stage, flag] of [ + [vendorDeepSeekNormalizeForChatCompletions, 'vendor-deepseek'], + [vendorKimiNormalizeForChatCompletions, 'vendor-kimi'], + ] as const) { + const { up } = await runStage(stage, 'response.chat.chatCompletions', { + 'request.chat.chatCompletions': chatCompletionsPayload([]), + 'route.attempt': attemptWith(flag), + }, { kind: 'stream' as const, frames: { [Symbol.asyncIterator]: () => (async function* () { throw failure; })() } }); + + await expect(drain(up['response.chat.chatCompletions'])).rejects.toBe(failure); + } + }); }); describe('the Messages interceptors, as stages', () => { @@ -666,6 +1104,56 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => const frames = await drain(up['response.chat.responses']); expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 150 }); }); + + // The Chat Completions wire's own array, which is not one family's: every source protocol + // that reaches an upstream over this endpoint runs it. What the order states is one thing in + // each direction. Going down, the usage chunk is asked for before any vendor can rewrite the + // body. Coming back, the vendor dialect has the first say — so the fold reads a cache count + // under OpenAI's name rather than DeepSeek's, and the carrier split reads the folded total. + it('assembles and runs the Chat Completions wire, the vendor dialects innermost', async () => { + const answer = streamOf([eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + // DeepSeek's own names, and the exclusive convention: 479 + 13312 + 373 = 14164. + usage: { + prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, + prompt_cache_hit_tokens: 13312, prompt_cache_miss_tokens: 479, + }, + } as unknown as ChatCompletionsStreamEvent)]); + const { down, up } = await runChain('response.chat.chatCompletions', [ + includeUsageStreamOptionsForChatCompletions, + normalizeUsageForChatCompletions, + applyRoleCompatibilityToChatCompletions, + normalizeExclusiveCachedTokensForChatCompletions, + vendorDeepSeekNormalizeForChatCompletions, + vendorQwenNormalizeForChatCompletions, + vendorKimiNormalizeForChatCompletions, + ], { + 'request.chat.chatCompletions': chatCompletionsPayload([ + { role: 'system', content: 'lead' }, + { role: 'user', content: 'hi' }, + { role: 'system', content: 'mid' }, + ], { reasoning_effort: 'none' }), + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user', 'vendor-deepseek'), + }, answer); + + const sent = down['request.chat.chatCompletions'] as Record; + expect(sent.stream_options).toEqual({ include_usage: true }); + expect((sent as unknown as ChatCompletionsPayload).messages.map(message => message.role)).toEqual(['system', 'user', 'user']); + // The sentinel this chain carried reached DeepSeek's dialect and left in DeepSeek's shape. + expect('reasoning_effort' in sent).toBe(false); + expect(sent.thinking).toEqual({ type: 'disabled' }); + + const frames = await drain(up['response.chat.chatCompletions']); + // Three rules ran on the way back, in the order that makes each one's input the one it + // expects: the vendor rename, then the fold, then the split onto a carrier. + expect(frames).toHaveLength(2); + expect('usage' in chatCompletionsEventOf(frames[0]!)).toBe(false); + expect(chatCompletionsUsageOf(frames[1]!)).toMatchObject({ + prompt_tokens: 13791, + prompt_tokens_details: { cached_tokens: 13312 }, + }); + }); }); const rolesOfPayload = (payload: Record): unknown[] => diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index 145318299a..7785e2594f 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -92,13 +92,17 @@ let affinityPayload: ChatCompletionsPayload = payload; const serve = async (wantsStream: boolean) => await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream); -const serveWith = async (gateway: ReturnType, wantsStream: boolean) => await run( +const serveWith = async ( + gateway: ReturnType, + wantsStream: boolean, + wantsUsageChunk = false, +) => await run( chatCompletionsServePipeline(payload), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], 'ingress.chat.sourceProtocol': 'chatCompletions', 'ingress.chat.chatCompletions.wantsStream': wantsStream, - 'ingress.chat.chatCompletions.wantsUsageChunk': false, + 'ingress.chat.chatCompletions.wantsUsageChunk': wantsUsageChunk, 'request.chat.chatCompletions': payload, 'serve.model': 'chat-model', }) as never, @@ -261,6 +265,125 @@ describe('the chat completions chain', () => { expect(outcome.failed).toBe(false); }); + // The test above hands over a usage chunk whatever was asked for, which is a stream no + // OpenAI-compatible upstream would have sent: the final usage chunk arrives only when + // `stream_options.include_usage` is on. This upstream behaves like a real one, so what it + // states is the whole billing path — a chain that stopped asking would meter a stream that + // reported nothing and bill the turn zero, with nothing else in the suite noticing. + it('asks the upstream for the usage chunk, and so bills a streaming turn at all', async () => { + let asked: { include_usage?: boolean } | null | undefined; + resolves([candidate(async (_model, body) => { + asked = (body as { stream_options?: { include_usage?: boolean } }).stream_options; + return asked?.include_usage === true + ? stream(chunk('hi'), { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [], usage: { prompt_tokens: 11, completion_tokens: 4, total_tokens: 15 }, + } as unknown as ChatCompletionsStreamEvent) + : stream(chunk('hi')); + })]); + + // The client did not ask to see the chunk, which is the ordinary case and the one where + // the two questions come apart: the upstream is asked for it regardless. + const { facts } = await serve(true); + const frames: SseFrame[] = []; + for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); + const outcome = await facts['response.chat.chatCompletions.streamedUsage']!; + + expect(asked).toEqual({ include_usage: true }); + expect(outcome.billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); + // And the client, who asked for none of this, is shown none of it. + expect(frames.some(frame => frame.data !== '[DONE]' && 'usage' in (JSON.parse(frame.data) as object))).toBe(false); + }); + + // The wire's response-direction rules, on the wire that owns them, in the order it composes + // them — which is a property of the array in `chatCompletionsWire` and of nothing else. A + // DeepSeek-shaped upstream reporting the exclusive cache convention exercises all three at + // once, and each one's input is what the one below it left: the vendor rename, then the fold + // that reads the renamed field, then the split that puts the settled usage on a carrier. + it('serves a vendor dialect through the wire-s rules, in the order the wire composes them', async () => { + resolves([candidate(async () => stream( + { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [{ index: 0, delta: { reasoning_content: 'pondering' }, finish_reason: null }], + } as unknown as ChatCompletionsStreamEvent, + { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + // Hung off the final delta rather than sent on a carrier, in DeepSeek's own cache + // field names, counting the cached prefix outside the input total: 479 + 13312 + 373 + // is the 14164 it states. + choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: 'stop' }], + usage: { + prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, + prompt_cache_hit_tokens: 13312, prompt_cache_miss_tokens: 479, + }, + } as unknown as ChatCompletionsStreamEvent, + ), 'up_a', ['vendor-deepseek'])]); + + const { facts } = await serveWith(mockChatGatewayCtx({ wantsStream: true }), true, true); + const written: ChatCompletionsStreamEvent[] = []; + for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { + if (frame.data !== '[DONE]' && affinityCarrier(frame) === undefined) { + written.push(JSON.parse(frame.data) as ChatCompletionsStreamEvent); + } + } + + // The vendor's reasoning field left under the canonical name. + expect(written[0]!.choices[0]!.delta).toEqual({ reasoning_text: 'pondering' }); + // The generated text is still the client's, whatever the rules did around it. + expect(written.some(event => event.choices[0]?.delta.content === 'hi')).toBe(true); + // Exactly one frame carries usage, and it is a carrier rather than a chunk that also + // carried a delta — which is the split, and it happened once. + const carriers = written.filter(event => 'usage' in event) as unknown as { choices: unknown[]; usage: Record }[]; + expect(carriers).toHaveLength(1); + expect(carriers[0]!.choices).toEqual([]); + // In OpenAI's names, with the cached prefix folded back into the input total the way + // OpenAI's contract states it: neither would be true if either rule ran out of turn. + expect(carriers[0]!.usage).toEqual({ + prompt_tokens: 13791, + completion_tokens: 373, + total_tokens: 14164, + prompt_tokens_details: { cached_tokens: 13312 }, + }); + }); + + // The other two dialects the wire composes, each stating it in a different direction: Qwen + // rewrites the request, Kimi the usage that comes back. The flags are independent, so one + // candidate carrying both is what runs the pair in one turn — an upstream in the field + // carries one of them. + it('serves the request-side and response-side vendor dialects the wire also composes', async () => { + let sent: Record | undefined; + affinityPayload = { ...payload, reasoning_effort: 'none' } as ChatCompletionsPayload; + resolves([candidate(async (_model, body) => { + sent = body as Record; + return stream(chunk('hi'), { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [], usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, cached_tokens: 30 }, + } as unknown as ChatCompletionsStreamEvent); + }, 'up_a', ['vendor-qwen', 'vendor-kimi'])]); + + const { facts } = await serveWith(mockChatGatewayCtx({ wantsStream: true }), true, true); + const written: ChatCompletionsStreamEvent[] = []; + for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { + if (frame.data !== '[DONE]' && affinityCarrier(frame) === undefined) { + written.push(JSON.parse(frame.data) as ChatCompletionsStreamEvent); + } + } + affinityPayload = payload; + + // Qwen's own spelling of "no reasoning" went out on the body this wire sent. + expect('reasoning_effort' in sent!).toBe(false); + expect(sent!.enable_thinking).toBe(false); + // And Kimi's flat cached count came back under the name every rule above it reads. The + // totals say the cache was counted inside the input here, so nothing is folded. + const carrier = written.find(event => 'usage' in event) as unknown as { usage: Record }; + expect(carrier.usage).toEqual({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + prompt_tokens_details: { cached_tokens: 30 }, + }); + }); + // Time to first token is measured where the token is — an envelope frame is not one, and a // run that never stamps it is recorded as having produced nothing to time. it('stamps the first frame that carried a generated token', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 9f513f0159..94412d8cf7 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -444,6 +444,45 @@ describe('a rule that speaks about one protocol-s wire', () => { await drain(); }); + // The other direction, and the one that pays for the whole arrangement: the usage chunk is + // asked for by the Chat Completions wire, so a Messages turn that reaches an upstream over + // that endpoint is metered like any other turn on it. A rule left in the Chat Completions + // *source* chain would ask for nothing here, and this turn would bill zero. + it('asks for the usage chunk on a turn that arrived from another protocol', async () => { + let sent: { stream_options?: unknown } | undefined; + resolves([candidate('up_a', { chatCompletions: {} }, { + callChatCompletions: async (_model, body) => { + sent = body as { stream_options?: unknown }; + return chatCompletionsTurn('hello'); + }, + })]); + + const { drain } = await serveMessages(); + + expect(sent?.stream_options).toEqual({ include_usage: true }); + await drain(); + }); + + // And a rule the *other* wire owns stays off this turn. `stream_options` is a Chat + // Completions field, so a Chat Completions turn dialled over Messages must not carry it — + // which is what the interceptor form said with `ctx.targetApi !== 'chat-completions'` and + // what position says here. + it('does not carry another wire-s request rules onto the wire it dialled', async () => { + let sent: Record | undefined; + resolves([candidate('up_a', { messages: {} }, { + callMessages: async (_model, body) => { + sent = body as Record; + return messagesTurn('hello'); + }, + })]); + + const { drain } = await serveChatCompletions(); + + expect(sent).toBeDefined(); + expect('stream_options' in sent!).toBe(false); + await drain(); + }); + // The other half, and this one assembly says rather than a run: the wire's own rules read // the wire's own request key, so `compose` refuses to place them *below* a handoff that has // taken it. It is one array's arrangement and not a proof — `compose` walks one array, and diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 6be6bc8fdc..7b471ba3b5 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -37,7 +37,13 @@ import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { applyRoleCompatibilityToChatCompletions, disableReasoningOnForcedToolChoiceForChatCompletions, + includeUsageStreamOptionsForChatCompletions, + normalizeExclusiveCachedTokensForChatCompletions, + normalizeUsageForChatCompletions, stripPromptCacheKeyForChatCompletions, + vendorDeepSeekNormalizeForChatCompletions, + vendorKimiNormalizeForChatCompletions, + vendorQwenNormalizeForChatCompletions, } from '../interceptors.ts'; import { messagesWire } from '../messages/pipeline.ts'; import { responsesWire } from '../responses/pipeline.ts'; @@ -282,11 +288,30 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< * Every source protocol that reaches an upstream over this endpoint runs this, whether the * client spoke Chat Completions or a handoff arrived here — which is what makes a rule that * speaks about *this* wire belong here. The role rewrite states what an upstream's Chat - * Completions endpoint accepts, so it applies to whatever body this wire actually sends and - * to nothing that leaves for another protocol. + * Completions endpoint accepts; the usage rules speak about the usage this wire asks for and + * this wire reports; the vendor dialects are how one upstream spells both. All of them apply + * to whatever body this wire actually sends and to nothing that leaves for another protocol. + * + * The order is the one the rules had as an onion, which is the same order in both directions: + * a stage earlier in the array rewrites the request first and reads the answer last. So the + * usage chunk is asked for above everything, and coming back the vendor dialects have the + * first say — the generic rules above them then read a body already in OpenAI-canonical form, + * with the cache-bucket fold seeing cache fields under OpenAI's names and the carrier split + * seeing usage the fold has already settled. + * + * `disableReasoningOnForcedToolChoice` and `stripPromptCacheKey` sit above the fork rather + * than here, and both still precede every vendor dialect on the request path: the canonical + * sentinel is emitted before a vendor spells it, and the field an upstream would reject is + * gone before a vendor rewrites what is left. */ export const chatCompletionsWire = (streamedUsage: string): readonly Stage[] => [ + includeUsageStreamOptionsForChatCompletions, + normalizeUsageForChatCompletions, applyRoleCompatibilityToChatCompletions, + normalizeExclusiveCachedTokensForChatCompletions, + vendorDeepSeekNormalizeForChatCompletions, + vendorQwenNormalizeForChatCompletions, + vendorKimiNormalizeForChatCompletions, callChatCompletionsUpstream(streamedUsage), ]; diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index ad6e7b920a..bec66cccc9 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -18,12 +18,17 @@ // learned on the way down carried in the stage's own closure, because a response-side `needs` // can only name what the ending provides. -import type { Chat } from './facts.ts'; +import type { Chat, ChatAnswer } from './facts.ts'; import { asJsonObject, type JsonObject, readJsonNumber } from '../../shared/json-helpers.ts'; import { isFailure, type AttemptSelector } from '../pipeline/facts.ts'; import { foldsExclusiveCacheTokens } from '../shared/telemetry/usage.ts'; import { defineStage, move, transform } from '@floway-dev/pipeline'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; +import type { + ChatCompletionsMessage, + ChatCompletionsPayload, + ChatCompletionsReasoningItem, + ChatCompletionsStreamEvent, +} from '@floway-dev/protocols/chat-completions'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { GeminiContent, GeminiPayload, GeminiStreamEvent, GeminiToolGroup } from '@floway-dev/protocols/gemini'; import type { MessagesPayload } from '@floway-dev/protocols/messages'; @@ -51,6 +56,106 @@ const mapKeepingIdentity = (items: T[], rewrite: (item: T) => T): T[] => { return changed ? mapped : items; }; +/** The response half of every rule here that rewrites a stream: a refusal and a collected + * body have no frames to rewrite, and telling the three apart is reading a value rather than + * dispatching on a declaration. `null` says there was nothing to do, so the caller hands the + * record it was given straight on. */ +const answerWithFrames = ( + answer: ChatAnswer, + rewrite: (frames: AsyncIterable>) => AsyncGenerator>, +): ChatAnswer | null => { + if (isFailure(answer) || answer.kind !== 'stream') return null; + const frames = answer.frames as AsyncIterable>; + return { kind: 'stream' as const, frames: { [Symbol.asyncIterator]: () => rewrite(frames) } }; +}; + +/** A stream with each event rewritten one for one. The frame comes back by identity when the + * rewrite changed nothing, which is the same conditional-write convention the request-side + * rules follow — and a transport frame is not an event, so it rides through untouched. */ +const rewritingEvents = async function* ( + frames: AsyncIterable>, + rewrite: (event: Event) => Event, +): AsyncGenerator> { + for await (const frame of frames) { + if (frame.type !== 'event') { + yield frame; + continue; + } + const event = rewrite(frame.event); + yield event === frame.event ? frame : eventFrame(event); + } +}; + +/** Where a protocol keeps the numbers the cache-bucket fold reads. The names are the whole of + * what a protocol contributes: the decision, the two contradictions it raises and the + * arithmetic are one rule, and it is written once below. */ +interface CacheBucketNames { + readonly input: string; + readonly output: string; + readonly details: string; + /** Read in the order given, because an upstream projecting Anthropic's cache-write bucket + * onto an OpenAI-shaped usage block may spell it either way. */ + readonly cacheWrite: readonly string[]; +} + +/** + * Restores OpenAI's inclusive input-token contract on a usage block that reports the cache + * buckets alongside the input total instead of inside it. + * + * OpenAI states the subset relationship outright — "Cached tokens here are counted as a subset + * of input tokens, meaning input tokens will include cached and uncached tokens" + * (https://github.com/openai/openai-openapi/blob/d4fb706e6e05d4cc9f1b33ca59b6e4f3e8edd439/openapi.yaml#L51043-L51049) + * — and every downstream consumer here subtracts on that basis. Anthropic takes the opposite + * convention: `input_tokens` counts only what was neither read from nor written to the cache, + * and the three buckets sum to the real input + * (https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching). + * + * A gateway that fronts an Anthropic-shaped upstream and projects it into an OpenAI shape can + * carry the exclusive convention through to a wire that declares the inclusive one. Portkey + * does exactly that — it assigns Anthropic's `input_tokens` straight to `prompt_tokens` while + * summing `total_tokens` from all four buckets + * (https://github.com/Portkey-AI/gateway/blob/669825cbe89ee51569918b8f78a9db486fd69dd4/src/providers/anthropic/chatComplete.ts#L612-L627). + * Charm Hyper produces the same shape by subtracting the cached prefix out of the input total + * it received: for one observed kimi-k3 turn it reported `prompt_tokens 479, cached_tokens + * 13312, completion_tokens 373, total_tokens 14164`, where 479 + 13312 + 373 = 14164. + * + * `foldsExclusiveCacheTokens` owns the decision and the two contradictions that must not pass + * silently; the `usage-exclusive-cached-tokens` flag is its declaration input. `total_tokens` + * itself is left alone: under the exclusive convention it already counts the real input, so + * the rewritten input plus output is what it was equal to all along. A block with nothing to + * fold comes back by identity. + */ +const withCacheBucketsFolded = ( + usage: JsonObject, + names: CacheBucketNames, + declaredExclusive: boolean, + identity: string, +): JsonObject => { + const inputTokens = readJsonNumber(usage[names.input]); + const outputTokens = readJsonNumber(usage[names.output]); + if (inputTokens == null || outputTokens == null) return usage; + + const details = asJsonObject(usage[names.details]); + const cacheRead = readJsonNumber(details?.cached_tokens) ?? 0; + const cacheWrite = names.cacheWrite.map(name => readJsonNumber(details?.[name])).find(value => value != null) ?? 0; + if (cacheRead === 0 && cacheWrite === 0) return usage; + + const folds = foldsExclusiveCacheTokens(declaredExclusive, { + inputTokens, + outputTokens, + totalTokens: readJsonNumber(usage.total_tokens) ?? undefined, + cacheRead, + cacheWrite, + }, identity); + if (!folds) return usage; + + return { ...usage, [names.input]: inputTokens + cacheRead + cacheWrite }; +}; + +/** How the attempt whose usage is being read is named in the errors the fold raises: an + * operator is told which upstream and which model to set the flag on. */ +const attemptIdentity = (attempt: AttemptSelector): string => `${attempt.upstreamId}/${attempt.modelId}`; + /** Role rewrites, in the fixed order `system → developer → system → user`. Later rewrites * are authoritative when flags overlap, and the last step affects only a system message * that appears after the leading run — which is what "mid-conversation" means. The fold @@ -217,6 +322,442 @@ export const stripPromptCacheKeyForChatCompletions = defineStage< })), }); +/** + * Asks the upstream for the usage chunk that billing is read off. + * + * Chat Completions emits the final usage-only chunk only when `stream_options.include_usage` + * is on, and the gateway meters every stream from those frames — so what the client asked for + * and what the upstream is asked for differ here, which is the one thing this rule exists to + * say. Nothing downstream has to thread the client's original value through, because it is a + * fact of its own: `ingress.chat.chatCompletions.wantsUsageChunk` describes the request that + * arrived and survives this rewrite, and the edge reads it to decide who is shown the chunk. + * + * It belongs to the wire rather than to the source chain because it names a field of *this* + * protocol's request. Every source protocol that reaches an upstream over this endpoint runs + * it — a Messages or Gemini turn served here would otherwise be metered off a stream that was + * never asked to report anything — and a turn that leaves for another wire gets that wire's + * own rule for the same thing. + * + * Reference: https://platform.openai.com/docs/api-reference/chat/create + */ +export const includeUsageStreamOptionsForChatCompletions = defineStage< + Chat<'request.chat.chatCompletions'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'includeUsageStreamOptions', + through: { + request: { + needs: ['request.chat.chatCompletions'], + consumes: [], + provides: ['request.chat.chatCompletions'], + }, + response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.chatCompletions'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => ({ + request: facts => { + const payload = facts['request.chat.chatCompletions']; + if (payload.stream_options?.include_usage === true) return facts; + // Whatever else the client put on `stream_options` is its own and stays; only the one + // field the gateway has an interest in is written. + return { + ...facts, + 'request.chat.chatCompletions': move({ + ...payload, + stream_options: { ...payload.stream_options, include_usage: true }, + }), + }; + }, + })), +}); + +/** + * Puts a usage block back on the carrier chunk the spec says it arrives on. + * + * OpenAI puts the final `usage` on a `choices: []` chunk of its own + * (https://platform.openai.com/docs/api-reference/chat-streaming), and some upstreams have + * been observed to attach it to the same chunk that carries the last delta and + * `finish_reason`. Such a chunk is split in two — the delta as it arrived, then a synthesized + * carrier holding the usage — so everything downstream can rely on the standard shape. + * + * It runs above the vendor normalizers, so on the way back it sees a usage block whose cache + * fields already carry OpenAI's names. + */ +export const normalizeUsageForChatCompletions = defineStage< + Record, + Record, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'normalizeUsage', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.chat.chatCompletions'], + consumes: [], + provides: ['response.chat.chatCompletions'], + }, + }, + execute: transform< + Record, + Record, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => ({ + response: facts => { + const answer = answerWithFrames(facts['response.chat.chatCompletions'], withUsageOnItsOwnCarrier); + return answer === null ? facts : { ...facts, 'response.chat.chatCompletions': move(answer) }; + }, + })), +}); + +/** One chunk in, two out — and only for the chunk that carried both, which is why this is not + * a one-for-one event rewrite. */ +const withUsageOnItsOwnCarrier = async function* ( + frames: AsyncIterable>, +): AsyncGenerator> { + for await (const frame of frames) { + if (frame.type !== 'event') { + yield frame; + continue; + } + const chunk = frame.event; + if (asJsonObject(chunk.usage) === null || chunk.choices.length === 0) { + yield frame; + continue; + } + const { usage, ...withoutUsage } = chunk; + yield eventFrame(withoutUsage); + yield eventFrame({ ...withoutUsage, choices: [], usage }); + } +}; + +/** + * Restores OpenAI's inclusive input-token contract on an upstream that reports the cache + * buckets alongside `prompt_tokens` instead of inside it. The fold, the evidence for the two + * conventions and the contradictions it raises are at `withCacheBucketsFolded`. + * + * Unconditional on this wire rather than flag-gated: `foldsExclusiveCacheTokens` reads + * `total_tokens` as the witness, and the `usage-exclusive-cached-tokens` flag is a declaration + * input for the responses whose totals witness nothing. + * + * Everything it says is about one upstream's Chat Completions wire — the flag it reads + * describes how that upstream writes its usage there, and the remedy its errors name is a + * setting for that upstream. That is why it belongs to the wire: on any other wire these + * events are a projection of some other protocol, where the flag answers a question about + * counts it does not describe and telling an operator to set it would be advice that cannot + * help. Nothing is lost by standing down there — a translator emits the canonical form, which + * is the one case the fold has nothing to do with. + * + * It runs below the vendor normalizers, so on the way back it reads usage whose cache fields + * already carry OpenAI's names. + */ +export const normalizeExclusiveCachedTokensForChatCompletions = defineStage< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'normalizeExclusiveCachedTokens', + through: { + request: { needs: ['route.attempt'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.chatCompletions'], + consumes: [], + provides: ['response.chat.chatCompletions'], + }, + }, + execute: transform< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => { + // Which attempt this is gets read on the way down and spoken about on the way back, which + // is the order `transform` runs the two halves in. + let attempt!: AttemptSelector; + return { + request: facts => { + attempt = facts['route.attempt']; + return facts; + }, + response: facts => { + const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); + const identity = attemptIdentity(attempt); + const answer = answerWithFrames( + facts['response.chat.chatCompletions'], + frames => rewritingEvents(frames, chunk => foldChatCompletionsUsage(chunk, declaredExclusive, identity)), + ); + return answer === null ? facts : { ...facts, 'response.chat.chatCompletions': move(answer) }; + }, + }; + }), +}); + +/** Chat Completions carries usage on the chunk's own root and names the buckets + * `prompt_tokens` and `prompt_tokens_details.{cached_tokens, cache_creation_input_tokens}`. */ +const CHAT_COMPLETIONS_CACHE_BUCKETS: CacheBucketNames = { + input: 'prompt_tokens', + output: 'completion_tokens', + details: 'prompt_tokens_details', + cacheWrite: ['cache_creation_input_tokens', 'cache_write_tokens'], +}; + +const foldChatCompletionsUsage = ( + chunk: ChatCompletionsStreamEvent, + declaredExclusive: boolean, + identity: string, +): ChatCompletionsStreamEvent => { + const usage = asJsonObject(chunk.usage); + if (usage === null) return chunk; + const folded = withCacheBucketsFolded(usage, CHAT_COMPLETIONS_CACHE_BUCKETS, declaredExclusive, identity); + return folded === usage ? chunk : { ...chunk, usage: folded as unknown as ChatCompletionsStreamEvent['usage'] }; +}; + +// ── Chat Completions vendor dialects ────────────────────────────────────────────────────── +// +// Each of the three below is last among this wire's rewrites, which is what gives it the final +// say on the outbound body and the first say on the inbound stream: everything above deals +// only in OpenAI-canonical form. The vendor flags are mutually exclusive in practice, but the +// stages are independent and run in the order they are composed if more than one is somehow on. + +/** + * DeepSeek's wire dialect. + * + * Outbound: `reasoning_effort: 'none'` is the gateway's canonical "no reasoning" sentinel and + * is not in DeepSeek's own enum, which uses a top-level `thinking: { type: 'disabled' }` + * instead. Assistant messages carry their reasoning on the scalar `reasoning_content` DeepSeek + * documents — it is the only field it reads, and it reports 400s when the assistant-message + * replay of a multi-turn tool-call loop omits it. And `response_format: { type: 'json_schema' }` + * is downgraded to `json_object`, which is the only structured output DeepSeek supports; the + * schema body is dropped rather than rejected by the upstream. + * + * Inbound: `reasoning_content` deltas become `reasoning_text`, and the `prompt_cache_hit_tokens` + * / `prompt_cache_miss_tokens` pair becomes OpenAI's `prompt_tokens_details.cached_tokens` — + * computed from the hit count alone, which is the cached prefix length. + * + * References: + * - https://api-docs.deepseek.com/zh-cn/guides/thinking_mode + * - https://api-docs.deepseek.com/guides/kv_cache + * - https://api-docs.deepseek.com/quick_start/agent_integrations/oh_my_pi + */ +export const vendorDeepSeekNormalizeForChatCompletions = defineStage< + Chat<'request.chat.chatCompletions' | 'route.attempt'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'vendorDeepSeekNormalize', + through: { + request: { + needs: ['request.chat.chatCompletions', 'route.attempt'], + consumes: [], + provides: ['request.chat.chatCompletions'], + }, + response: { + needs: ['response.chat.chatCompletions'], + consumes: [], + provides: ['response.chat.chatCompletions'], + }, + }, + execute: transform< + Chat<'request.chat.chatCompletions' | 'route.attempt'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => { + // Whether this vendor's dialect applies at all is read on the way down and acted on in + // both directions, which is the order `transform` runs the two halves in. + let enabled = false; + return { + request: facts => { + enabled = facts['route.attempt'].flags.includes('vendor-deepseek'); + if (!enabled) return facts; + const payload = facts['request.chat.chatCompletions']; + const normalized = deepSeekOutbound(payload); + return normalized === payload ? facts : { ...facts, 'request.chat.chatCompletions': move(normalized) }; + }, + response: facts => { + if (!enabled) return facts; + const answer = answerWithFrames( + facts['response.chat.chatCompletions'], + frames => rewritingEvents(frames, chunk => deepSeekInboundUsage(deepSeekInboundDeltas(chunk))), + ); + return answer === null ? facts : { ...facts, 'response.chat.chatCompletions': move(answer) }; + }, + }; + }), +}); + +const deepSeekOutbound = (payload: ChatCompletionsPayload): ChatCompletionsPayload => { + const withThinking = payload.reasoning_effort === 'none' + ? { ...withoutKeys(payload, ['reasoning_effort']), thinking: { type: 'disabled' } } satisfies ChatCompletionsPayloadWithDeepSeekThinking + : payload; + const withResponseFormat = withThinking.response_format?.type === 'json_schema' + ? { ...withThinking, response_format: { type: 'json_object' } } + : withThinking; + const messages = mapKeepingIdentity(withResponseFormat.messages, deepSeekAssistantReasoning); + return messages === withResponseFormat.messages ? withResponseFormat : { ...withResponseFormat, messages }; +}; + +/** The reasoning DeepSeek reads, and only that: `reasoning_opaque` is the OpenAI-canonical + * signature for cross-turn replay and DeepSeek does not accept it, so it goes with the two + * fields that are projected onto `reasoning_content`. */ +const deepSeekAssistantReasoning = (message: ChatCompletionsMessage): ChatCompletionsMessage => { + const stripped = withoutKeys(message, ['reasoning_text', 'reasoning_opaque', 'reasoning_items']); + if (stripped === message) return message; + const text = typeof message.reasoning_text === 'string' + ? message.reasoning_text + : deepSeekReasoningFromItems(message.reasoning_items); + if (text === undefined) return stripped; + const projected: ChatCompletionsMessageWithDeepSeekReasoning = { ...stripped, reasoning_content: text }; + return projected; +}; + +/** The newer OpenAI shape carries reasoning as summary items; DeepSeek documents only the + * scalar, so what summaries there are become it. */ +const deepSeekReasoningFromItems = (items: ChatCompletionsReasoningItem[] | null | undefined): string | undefined => { + const parts = items?.flatMap(item => item.summary?.map(summary => summary.text) ?? []) ?? []; + return parts.length > 0 ? parts.join('') : undefined; +}; + +const deepSeekInboundDeltas = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { + const choices = mapKeepingIdentity(chunk.choices, choice => { + const delta = choice.delta as ChatCompletionsDeltaWithDeepSeekReasoning; + if (typeof delta.reasoning_content !== 'string') return choice; + const stripped = withoutKeys(delta, ['reasoning_content']); + return { + ...choice, + delta: delta.reasoning_text === undefined ? { ...stripped, reasoning_text: delta.reasoning_content } : stripped, + }; + }); + return choices === chunk.choices ? chunk : { ...chunk, choices }; +}; + +/** DeepSeek's "hit" count is the cached prefix length, which is what OpenAI's + * `cached_tokens` means; the "miss" count is what is left of the input and is dropped. */ +const DEEPSEEK_CACHE_FIELDS = ['prompt_cache_hit_tokens', 'prompt_cache_miss_tokens'] as const; + +const deepSeekInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { + const usage = asJsonObject(chunk.usage); + if (usage === null) return chunk; + const stripped = withoutKeys(usage, DEEPSEEK_CACHE_FIELDS); + if (stripped === usage) return chunk; + const hit = readJsonNumber(usage.prompt_cache_hit_tokens); + const next: JsonObject = hit == null + ? stripped + : { ...stripped, prompt_tokens_details: { ...(asJsonObject(usage.prompt_tokens_details) ?? {}), cached_tokens: hit } }; + return { ...chunk, usage: next as unknown as ChatCompletionsStreamEvent['usage'] }; +}; + +/** Qwen says "no reasoning" with a top-level `enable_thinking: false` rather than with the + * canonical sentinel. Its response shape matches OpenAI for the fields the gateway reads, so + * there is nothing to do on the way back. + * https://www.alibabacloud.com/help/en/model-studio/deep-thinking */ +export const vendorQwenNormalizeForChatCompletions = defineStage< + Chat<'request.chat.chatCompletions' | 'route.attempt'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'vendorQwenNormalize', + through: { + request: { + needs: ['request.chat.chatCompletions', 'route.attempt'], + consumes: [], + provides: ['request.chat.chatCompletions'], + }, + response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.chatCompletions' | 'route.attempt'>, + Chat<'request.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('vendor-qwen')) return facts; + const payload = facts['request.chat.chatCompletions']; + if (payload.reasoning_effort !== 'none') return facts; + const normalized: ChatCompletionsPayloadWithQwenThinking = { + ...withoutKeys(payload, ['reasoning_effort']), + enable_thinking: false, + }; + return { ...facts, 'request.chat.chatCompletions': move(normalized) }; + }, + })), +}); + +/** Kimi (Moonshot) reports the cached prefix on a flat `cached_tokens` beside the totals; the + * rules above this one read OpenAI's `prompt_tokens_details.cached_tokens`, so it is put + * there. Kimi accepts the canonical request shape, so there is nothing to do on the way out. + * https://platform.kimi.com/docs/api/chat */ +export const vendorKimiNormalizeForChatCompletions = defineStage< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> +>({ + name: 'vendorKimiNormalize', + through: { + request: { needs: ['route.attempt'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.chatCompletions'], + consumes: [], + provides: ['response.chat.chatCompletions'], + }, + }, + execute: transform< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.chatCompletions'>, + Chat<'response.chat.chatCompletions'> + >(() => { + let enabled = false; + return { + request: facts => { + enabled = facts['route.attempt'].flags.includes('vendor-kimi'); + return facts; + }, + response: facts => { + if (!enabled) return facts; + const answer = answerWithFrames( + facts['response.chat.chatCompletions'], + frames => rewritingEvents(frames, kimiInboundUsage), + ); + return answer === null ? facts : { ...facts, 'response.chat.chatCompletions': move(answer) }; + }, + }; + }), +}); + +const kimiInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { + const usage = asJsonObject(chunk.usage); + if (usage === null) return chunk; + const cached = readJsonNumber(usage.cached_tokens); + if (cached == null) return chunk; + const next: JsonObject = { + ...withoutKeys(usage, ['cached_tokens']), + prompt_tokens_details: { ...(asJsonObject(usage.prompt_tokens_details) ?? {}), cached_tokens: cached }, + }; + return { ...chunk, usage: next as unknown as ChatCompletionsStreamEvent['usage'] }; +}; + +/** None of the three fields is a Chat Completions field, so each is declared beside the vendor + * that reads it rather than widening the protocol's own types. */ +type ChatCompletionsPayloadWithDeepSeekThinking = Omit & { thinking: { type: string } }; +type ChatCompletionsPayloadWithQwenThinking = Omit & { enable_thinking: false }; +type ChatCompletionsMessageWithDeepSeekReasoning = + Omit & { reasoning_content: string }; +type ChatCompletionsDeltaWithDeepSeekReasoning = + ChatCompletionsStreamEvent['choices'][number]['delta'] & { reasoning_content?: unknown }; + // ── Messages ────────────────────────────────────────────────────────────────────────────── /** @@ -749,8 +1290,8 @@ export const stripPromptCacheKeyForResponses = defineStage< * flag is a declaration input for the responses whose totals witness nothing. Every event * that carries a response resource repeats the whole resource, so the rewrite applies to each * of them rather than to a single terminal frame. The evidence for the two conventions and - * the contradictions that raise are documented at `foldsExclusiveCacheTokens` and at the Chat - * Completions normalizer of the same name. + * the contradictions that raise are documented at `foldsExclusiveCacheTokens` and at + * `withCacheBucketsFolded`, which is the fold itself. */ export const normalizeExclusiveCachedTokensForResponses = defineStage< Chat<'route.attempt'>, @@ -778,41 +1319,28 @@ export const normalizeExclusiveCachedTokensForResponses = defineStage< return facts; }, response: facts => { - const answer = facts['response.chat.responses']; - if (isFailure(answer) || answer.kind !== 'stream') return facts; - const frames = answer.frames as AsyncIterable>; const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); - const identity = `${attempt.upstreamId}/${attempt.modelId}`; - return { - ...facts, - 'response.chat.responses': move({ - kind: 'stream' as const, - frames: { [Symbol.asyncIterator]: () => withCacheTokensFolded(frames, declaredExclusive, identity) }, - }), - }; + const identity = attemptIdentity(attempt); + const answer = answerWithFrames( + facts['response.chat.responses'], + frames => rewritingEvents(frames, event => foldResponsesUsage(event, declaredExclusive, identity)), + ); + return answer === null ? facts : { ...facts, 'response.chat.responses': move(answer) }; }, }; }), }); -const withCacheTokensFolded = async function* ( - frames: AsyncIterable>, - declaredExclusive: boolean, - identity: string, -): AsyncGenerator> { - for await (const frame of frames) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = foldInboundUsage(frame.event, declaredExclusive, identity); - yield event === frame.event ? frame : eventFrame(event); - } -}; - /** Responses carries usage on `event.response.usage` and names the buckets `input_tokens` and * `input_tokens_details.{cached_tokens, cache_write_tokens}`. */ -const foldInboundUsage = ( +const RESPONSES_CACHE_BUCKETS: CacheBucketNames = { + input: 'input_tokens', + output: 'output_tokens', + details: 'input_tokens_details', + cacheWrite: ['cache_write_tokens'], +}; + +const foldResponsesUsage = ( event: ResponsesStreamEvent, declaredExclusive: boolean, identity: string, @@ -820,28 +1348,10 @@ const foldInboundUsage = ( if (!('response' in event)) return event; const response = asJsonObject(event.response); const usage = asJsonObject(response?.usage); - if (!response || !usage) return event; - const inputTokens = readJsonNumber(usage.input_tokens); - const outputTokens = readJsonNumber(usage.output_tokens); - if (inputTokens == null || outputTokens == null) return event; - - const details = asJsonObject(usage.input_tokens_details); - const cacheRead = readJsonNumber(details?.cached_tokens) ?? 0; - const cacheWrite = readJsonNumber(details?.cache_write_tokens) ?? 0; - if (cacheRead === 0 && cacheWrite === 0) return event; - - const fold = foldsExclusiveCacheTokens(declaredExclusive, { - inputTokens, - outputTokens, - totalTokens: readJsonNumber(usage.total_tokens) ?? undefined, - cacheRead, - cacheWrite, - }, identity); - if (!fold) return event; - - const nextUsage: JsonObject = { ...usage, input_tokens: inputTokens + cacheRead + cacheWrite }; - const nextResponse: JsonObject = { ...response, usage: nextUsage }; - return { ...event, response: nextResponse } as unknown as ResponsesStreamEvent; + if (response === null || usage === null) return event; + const folded = withCacheBucketsFolded(usage, RESPONSES_CACHE_BUCKETS, declaredExclusive, identity); + if (folded === usage) return event; + return { ...event, response: { ...response, usage: folded } as JsonObject } as unknown as ResponsesStreamEvent; }; /** DeepSeek says "no reasoning" with a top-level `thinking: { type: 'disabled' }` rather than From 248f89f2d832713b2d35c40bb628a5f9243c2fbb Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 11:30:16 +0800 Subject: [PATCH 052/156] fix(chat): shape a translated turn for the wire it lands on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The interceptor array was split by role: a rule a wire owns sat in its source chain, so a turn arriving over a translation ran one rule of its target's set instead of the whole thing. A DeepSeek upstream reached as responses via chat completions got neither the reasoning rewrite nor the cache-key strip, both of which that upstream answers 400 without. The old surface ran the whole target array on a translated body, and the human rejected the role split that would have justified anything else — 「这是同一个 数组」. A rule about a wire now lives in that wire's chain, which is also what the four dropped guards said: each keyed on the target, not on whether the turn had been translated. The test that proves it is the one nothing had: a turn arriving from another protocol, scrubbed by a rule its own source chain never ran. --- .../data-plane/chat/translated-wires_test.ts | 24 +++++++++++++++++++ .../src/data-plane/chat/messages/pipeline.ts | 4 ++-- .../src/data-plane/chat/responses/pipeline.ts | 8 +++---- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 94412d8cf7..829f3301bf 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -463,6 +463,30 @@ describe('a rule that speaks about one protocol-s wire', () => { await drain(); }); + // The same arrangement for a rule no guard ever covered. The old surface ran the *whole* + // target array on a translated body, so a turn arriving over a translation was shaped for + // the wire it landed on. A rule left in the Messages source chain would scrub nothing here. + it('applies the target wire-s unguarded rules to a turn that arrived translated', async () => { + let sent: { system?: unknown } | undefined; + resolves([candidate('up_a', { messages: {} }, { + callMessages: async (_model, body) => { + sent = body as { system?: unknown }; + return messagesTurn('hello'); + }, + }, ['strip-billing-attribution'])]); + + const { drain } = await serveChatCompletions({ + ...chatCompletionsPayload, + messages: [ + { role: 'system', content: 'be brief\nx-anthropic-billing-header: acct-42' }, + { role: 'user', content: 'hi' }, + ], + } as ChatCompletionsPayload); + + expect(String(JSON.stringify(sent?.system))).not.toContain('x-anthropic-billing-header'); + await drain(); + }); + // And a rule the *other* wire owns stays off this turn. `stream_options` is a Chat // Completions field, so a Chat Completions turn dialled over Messages must not carry it — // which is what the interceptor form said with `ctx.targetApi !== 'chat-completions'` and diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index a1ed26e6b6..02605365b2 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -354,6 +354,8 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< * that leaves for another protocol. */ export const messagesWire = (streamedUsage: string): readonly Stage[] => [ + stripBillingAttributionFromMessages, + disableReasoningOnForcedToolChoiceForMessages, applyRoleCompatibilityToMessages, callMessagesUpstream(streamedUsage), ]; @@ -497,8 +499,6 @@ export const messagesServePipeline = (payload: MessagesPayload): Pipeline Date: Mon, 17 Aug 2026 11:47:28 +0800 Subject: [PATCH 053/156] fix(chat): hand a provider a body it owns, and record the frames it served MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects with one root between them: what a chat turn hands across a boundary was the record's own value. A record is deep-frozen, and a provider's interceptors shape their payload in place — Copilot writes `copilot_cache_control` onto individual messages. Spreading a frozen fact gives a fresh top level over frozen children, so every Copilot chat turn answered 502 with `Cannot add property copilot_cache_control, object is not extensible`, raised where nothing could explain it. Building an attempt's body is now one boundary rather than three copies of the same three lines, and the copy it makes goes all the way down. The dump was the other half. `stream.frame` had no producer on any chat chain, so a key with retention configured paid for records holding stage boundaries and no content, and the dashboard's collected view had nothing to replay. The tee sits at the family's edge, where the frames are still protocol frames and are already the source protocol's — what the client read. A non-streaming turn folds those same frames into one value, so recording there is what puts the stream in the record beside the body assembled from it. The frame test is what found the 502: nothing had run a Copilot chat turn through a pipeline and read the answer. --- .../data-plane/chat/run-dump-frames_test.ts | 100 ++++++++++++++++++ .../data-plane/pipeline/attempt-body_test.ts | 67 ++++++++++++ .../chat/chat-completions/pipeline.ts | 19 ++-- .../src/data-plane/chat/gemini/pipeline.ts | 14 ++- .../src/data-plane/chat/messages/pipeline.ts | 23 ++-- .../src/data-plane/pipeline/attempt-body.ts | 37 +++++++ packages/gateway/src/dump/turn-dump.ts | 29 +++++ 7 files changed, 264 insertions(+), 25 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts create mode 100644 packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts create mode 100644 packages/gateway/src/data-plane/pipeline/attempt-body.ts diff --git a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts new file mode 100644 index 0000000000..0679642c8d --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts @@ -0,0 +1,100 @@ +// The frames a chat turn was served are in its run record. +// +// A run record that holds only stage boundaries describes the shape of the turn and none of +// its content, and the dashboard's collected view is assembled by replaying exactly these +// events. The tee sits at the family's edge, so what lands in the record is the source +// protocol's frames — what the client read — rather than whatever the upstream spoke. + +import { test, vi } from 'vitest'; + +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; +import { copilotModels, flushAsyncWork, requestApp, setupAppTest, sseChatCompletionsResponse } from '../../test-utils/app.ts'; +import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const withChatCompletionsUpstream = async (run: () => Promise): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') { + return jsonResponse(copilotModels([{ id: 'gpt-frames', supported_endpoints: ['/chat/completions'] }])); + } + if (url.pathname === '/chat/completions') { + return sseChatCompletionsResponse({ + id: 'chatcmpl_frames', + object: 'chat.completion', + model: 'gpt-frames', + choices: [{ index: 0, message: { role: 'assistant', content: 'hello there' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 3, completion_tokens: 5, total_tokens: 8 }, + }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + run, + ); + +// The run format interns every node it has seen, so a frame arrives as a reference into the +// table the `object` events build up. Resolving it here is what makes the assertion about the +// frame's content rather than about an event existing. +const resolveFrames = (events: readonly Record[]): unknown[] => { + const table = new Map(); + for (const event of events) { + if (event.type !== 'object') continue; + const from = event.fromObjectId as number; + (event.nodes as unknown[]).forEach((node, index) => { table.set(from + index, node); }); + } + const resolve = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(resolve); + if (typeof value !== 'object' || value === null) return value; + const record = value as Record; + if ('$' in record) return resolve(table.get(record.$ as number)); + return Object.fromEntries(Object.entries(record).map(([key, child]) => [key, resolve(child)])); + }; + return events + .filter(event => event.type === 'stream.frame') + .flatMap(event => (event.frames as unknown[]).map(resolve)); +}; + +const recordedFramesOf = async (stream: boolean): Promise => { + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + + await withChatCompletionsUpstream(async () => { + const response = await requestApp('/v1/chat/completions', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-frames', messages: [{ role: 'user', content: 'hi' }], stream }), + }); + assertEquals(response.status, 200); + // Reading is what records, so a test that never read the body would prove the opposite of + // what it claims. + await response.text(); + }); + + await flushAsyncWork(); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + const stored = dumps.stored[0]; + assertExists(stored); + return resolveFrames(eventsOf(runRecordOf(stored.record))); +}; + +test('a streamed chat turn records the frames it served', async () => { + const frames = await recordedFramesOf(true); + assertEquals(frames.length > 0, true); + // The content is the point, not the count: a record holding frames with nothing in them is + // the same empty record with more events. + assertEquals(JSON.stringify(frames).includes('hello there'), true); +}); + +test('a collected chat turn records the frames it was assembled from', async () => { + // 「流式请求前端看到流+前端组装结果,非流式请求前端看到流+原始组装结果」 — the frames are + // recorded either way, which is what lets the operator see the stream behind a single body. + const frames = await recordedFramesOf(false); + assertEquals(frames.length > 0, true); + assertEquals(JSON.stringify(frames).includes('hello there'), true); +}); diff --git a/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts b/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts new file mode 100644 index 0000000000..c79128b401 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts @@ -0,0 +1,67 @@ +// A provider is handed a body it owns. +// +// Provider interceptors shape their payload in place — Copilot marks individual messages for +// caching — and the record they are built from is deep-frozen. A copy that stopped at the top +// level would satisfy every type in the system and still throw at the dial, so the depth of it +// is the property worth pinning. + +import { describe, it } from 'vitest'; + +import { bodyForAttempt } from '../../../src/data-plane/pipeline/attempt-body.ts'; +import { move } from '@floway-dev/pipeline'; +import type { ModelCandidate } from '@floway-dev/provider'; +import { assertEquals } from '@floway-dev/test-utils'; + +const candidateFor = (id: string, rules?: ModelCandidate['rules']): ModelCandidate => + ({ model: { id }, ...(rules === undefined ? {} : { rules }) }) as ModelCandidate; + +const recordedRequest = () => { + const { request } = move({ + request: { + model: 'alias-name', + messages: [{ role: 'system', content: 'be brief' }, { role: 'user', content: 'hi' }], + }, + }); + return request; +}; + +describe('bodyForAttempt', () => { + it('hands over children a provider can write into', () => { + const body = bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), () => {}); + + // What the Copilot cache-control rule does, at the node it does it to. + const message = body.messages[0] as Record; + message.copilot_cache_control = { type: 'ephemeral' }; + assertEquals(message.copilot_cache_control, { type: 'ephemeral' }); + }); + + it('leaves the record it was built from untouched', () => { + const recorded = recordedRequest(); + const body = bodyForAttempt(recorded, candidateFor('upstream-model'), () => {}); + (body.messages[0] as Record).copilot_cache_control = { type: 'ephemeral' }; + + assertEquals('copilot_cache_control' in recorded.messages[0], false); + assertEquals(recorded.model, 'alias-name'); + }); + + it('addresses the model the candidate resolved and drops the one the client named', () => { + const body = bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), () => {}); + assertEquals('model' in body, false); + }); + + it('applies the alias rules over the copy', () => { + const rules = { reasoning: { effort: 'high' } } as NonNullable; + const body = bodyForAttempt( + recordedRequest(), + candidateFor('upstream-model', rules), + (payload, applied) => { (payload as Record).reasoning_effort = applied.reasoning?.effort; }, + ); + assertEquals((body as Record).reasoning_effort, 'high'); + }); + + it('applies nothing where the alias carries no rules', () => { + let ran = false; + bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), () => { ran = true; }); + assertEquals(ran, false); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 7b471ba3b5..25d22da0c9 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -23,6 +23,8 @@ import { wrapChatCompletionsAffinityEgress } from './affinity/egress.ts'; import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; import { billableUsageFromChatCompletionsEvent } from './usage.ts'; +import { recordFrames } from '../../../dump/turn-dump.ts'; +import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; @@ -144,9 +146,12 @@ const emitChatCompletions = defineStage< // carrying it comes back to the upstream that issued it. This is the other half of the // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. - const frames = wrapChatCompletionsAffinityEgress( - answer.frames as AsyncIterable>, - affinityEgressOptions(use.gateway), + const frames = recordFrames( + wrapChatCompletionsAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ), + use.gateway.dump, ); if (!back['ingress.chat.chatCompletions.wantsStream']) { return { @@ -214,12 +219,8 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< // What the record holds by now: the payload affinity materialized for this candidate, // as every stage between the fork and here has rewritten it — or, on a translated wire, - // what the handoff put here. The id the client addressed does not travel — the provider - // re-stamps whatever it resolved upstream — and an alias' own rules apply to the body - // that is sent. - const payload = { ...facts['request.chat.chatCompletions'], model: candidate.model.id }; - if (candidate.rules !== undefined) applyRulesToUpstreamChatCompletions(payload, candidate.rules); - const { model: _addressed, ...body } = payload; + // what the handoff put here. + const body = bodyForAttempt(facts['request.chat.chatCompletions'], candidate, applyRulesToUpstreamChatCompletions); let result; try { diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 60e3ce3b1b..34cfb21559 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -43,8 +43,10 @@ // the status the upstream sent is the status the client sees, with no coercion of the codes // that envelope maps to `INTERNAL`. +import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; import { analyzeGeminiAffinity } from './affinity/ingress.ts'; import { renderGeminiError } from './errors.ts'; +import { recordFrames } from '../../../dump/turn-dump.ts'; import { isFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; @@ -61,9 +63,8 @@ import { } from '../interceptors.ts'; import { messagesWire } from '../messages/pipeline.ts'; import { responsesWire } from '../responses/pipeline.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, transform, type Pipeline } from '@floway-dev/pipeline'; import { renderProtocolError, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; @@ -153,9 +154,12 @@ const emitGemini = defineStage< // carrying it comes back to the upstream that issued it. This is the other half of the // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. - const frames = wrapGeminiAffinityEgress( - answer.frames as AsyncIterable>, - affinityEgressOptions(use.gateway), + const frames = recordFrames( + wrapGeminiAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ), + use.gateway.dump, ); if (!back['ingress.chat.gemini.wantsStream']) { return { diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index 02605365b2..e3e13333e9 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -16,9 +16,13 @@ // `count-tokens.ts`. This family's own interceptors are not all stages yet either, so the // array between the materialized payload and the fork is short rather than complete. +import { wrapMessagesAffinityEgress } from './affinity/egress.ts'; import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { renderMessagesError } from './errors.ts'; +import { isClaudeCodeProbe, probeFrames } from './interceptors/answer-claude-code-probe.ts'; import { createMessagesBillableUsageReader } from './usage.ts'; +import { recordFrames } from '../../../dump/turn-dump.ts'; +import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; @@ -37,13 +41,11 @@ import { stripBillingAttributionFromMessages, } from '../interceptors.ts'; import { responsesWire } from '../responses/pipeline.ts'; +import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; -import { wrapMessagesAffinityEgress } from './affinity/egress.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; -import { isClaudeCodeProbe, probeFrames } from './interceptors/answer-claude-code-probe.ts'; import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; import { renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { @@ -129,9 +131,12 @@ const emitMessages = defineStage< // carrying it comes back to the upstream that issued it. This is the other half of the // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. - const frames = wrapMessagesAffinityEgress( - answer.frames as AsyncIterable>, - affinityEgressOptions(use.gateway), + const frames = recordFrames( + wrapMessagesAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ), + use.gateway.dump, ); if (!back['ingress.chat.messages.wantsStream']) { return { @@ -267,11 +272,7 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< // every stage between the fork and here has rewritten it — or, on a translated wire, what // the handoff put here. A thinking signature was rewritten for the upstream that will see // it, and one that no upstream but the issuer can read was dropped rather than sent on. - // The id the client addressed does not travel — the provider re-stamps whatever it - // resolved upstream — and an alias' own rules apply to the body that is sent. - const payload = { ...facts['request.chat.messages'], model: candidate.model.id }; - if (candidate.rules !== undefined) applyRulesToUpstreamMessages(payload, candidate.rules); - const { model: _addressed, ...body } = payload; + const body = bodyForAttempt(facts['request.chat.messages'], candidate, applyRulesToUpstreamMessages); // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump diff --git a/packages/gateway/src/data-plane/pipeline/attempt-body.ts b/packages/gateway/src/data-plane/pipeline/attempt-body.ts new file mode 100644 index 0000000000..9d750fa904 --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/attempt-body.ts @@ -0,0 +1,37 @@ +// What a provider is handed when a stage dials it. +// +// The two sides of this boundary disagree about ownership, and both are right. A record's +// values are frozen — a fact that could be edited after the event is not a fact — while a +// provider's interceptors shape the body they are given in place, down to nested nodes: +// Copilot writes `copilot_cache_control` onto individual messages, and its initiator rule +// writes into a nested field of a Messages payload. +// +// Spreading a frozen fact produces a fresh top level over frozen children, so those writes +// throw — as a 502 carrying `Cannot add property …, object is not extensible`, raised at the +// point in the stack least able to explain it. What crosses this boundary is therefore a copy +// the provider owns outright, children included. +// +// The copy costs one pass over the payload, beside the serialization the dial performs +// immediately afterwards. What it buys is that a provider may keep being written the way every +// provider here is already written. + +import type { ModelCandidate } from '@floway-dev/provider'; + +/** + * The body for this attempt: the record's request, copied for the provider, addressed to the + * model the candidate resolved, with the alias' own rules applied over it. + * + * The id the client addressed does not travel. An alias is a gateway concept — the provider + * re-stamps whatever it resolved upstream — so the key is dropped rather than forwarded, and + * the alias' rules apply to the body that is actually sent. + */ +export const bodyForAttempt = ( + recorded: T, + candidate: ModelCandidate, + applyRules: (body: T, rules: NonNullable) => void, +): Omit => { + const payload = { ...structuredClone(recorded), model: candidate.model.id } as T; + if (candidate.rules !== undefined) applyRules(payload, candidate.rules); + const { model: _addressed, ...body } = payload; + return body; +}; diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts index 5e8483a4e6..50947aa110 100644 --- a/packages/gateway/src/dump/turn-dump.ts +++ b/packages/gateway/src/dump/turn-dump.ts @@ -23,3 +23,32 @@ export interface TurnDump { finalize(status: number | null, responseBytes: number | readonly unknown[]): void; finalize(response: Response): Response; } + +/** + * Records a stream's frames as they are read, and hands them on untouched. + * + * What the record is to hold is the stream the client was actually served, so the tee belongs + * at the family's edge: below it the frames are the upstream's and may still be translated, + * above it they are transport frames the record does not describe. A non-streaming turn folds + * the same frames into one value, and recording here is what puts both in the record — the + * frames as they flowed beside the value assembled from them. + * + * Reading is what records: a losing branch nobody read contributes nothing, and a stream that + * stopped short is recorded as far as it got. + */ +export const recordFrames = ( + frames: AsyncIterable>, + dump: TurnDump | null, +): AsyncIterable> => { + // No recording configured hands the same iterable back, so a record shows no step where + // nothing happened and the frames are not wrapped for nobody. + if (dump === null) return frames; + return { + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + dump.frame(frame); + yield frame; + } + })(), + }; +}; From 24c3d035c901f8b1acb197a899601515d42fb736 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 11:48:52 +0800 Subject: [PATCH 054/156] fix(chat): build a count_tokens dial through the same boundary as generation Counting reaches the same provider and runs the same interceptors, so it carried the same defect: a Copilot `/v1/messages/count_tokens` turn 502'd where the initiator rule wrote into a nested field of a frozen payload. The chain's own claim is that what it measures is what generation would send. Building its body the way generation builds it is what makes that true of the dial as well as of the rules. --- .../src/data-plane/chat/messages/count-tokens.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts index b7fac70a6a..32642a7296 100644 --- a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts @@ -29,6 +29,7 @@ import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { renderMessagesError } from './errors.ts'; import { prepareMessagesWebSearchShimRequest } from './interceptors/web-search-shim.ts'; import type { MessagesFacts } from './pipeline.ts'; +import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; @@ -181,12 +182,9 @@ const callMessagesCountTokensUpstream = defineStage< const candidate = use.resolveAttempt(facts['route.attempt']); // The payload affinity materialized for this candidate, as every stage between the fork - // and here has rewritten it. The id the client addressed does not travel — the provider - // re-stamps whatever it resolved upstream — and an alias' own rules apply to the body - // that is sent, so what is measured is what generation would be charged for. - const payload = { ...facts['request.chat.messages'], model: candidate.model.id }; - if (candidate.rules !== undefined) applyRulesToUpstreamMessages(payload, candidate.rules); - const { model: _addressed, ...body } = payload; + // and here has rewritten it — built for the dial exactly as generation builds it, so what + // is measured is what generation would be charged for. + const body = bodyForAttempt(facts['request.chat.messages'], candidate, applyRulesToUpstreamMessages); // Anthropic's beta flags have a typed path of their own, so no header allowlist can admit // them, and they are the client's own only when the client spoke this protocol. From 186c077f6ed84820613ead983dd87ee55ce7badf Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 11:55:03 +0800 Subject: [PATCH 055/156] fix(gateway): render a failure the same way at every edge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Failure` carries three things a client could be shown and each edge decided for itself which to reach for, so four of six read the upstream's body and the protocol's default and stepped over the envelope in between. Nothing is currently lost — only the Chat Completions and Responses refusals write one — but the field is part of the shape every edge receives, and the next refusal to carry one on Messages or Gemini would have disappeared silently. The order is a property of the type rather than of any family: the refusing party's own words first, then the gateway's own account of a refusal it made, then the protocol's rendering of a status and a sentence. Stated once, beside the type that raises the question. --- .../pipeline/failure-render_test.ts | 49 +++++++++++++++++++ .../chat/chat-completions/pipeline.ts | 10 ++-- .../data-plane/chat/gemini/count-tokens.ts | 5 +- .../src/data-plane/chat/gemini/pipeline.ts | 6 +-- .../data-plane/chat/messages/count-tokens.ts | 5 +- .../src/data-plane/chat/messages/pipeline.ts | 6 +-- .../gateway/src/data-plane/pipeline/facts.ts | 13 ++++- 7 files changed, 76 insertions(+), 18 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts diff --git a/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts b/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts new file mode 100644 index 0000000000..cec6d12767 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts @@ -0,0 +1,49 @@ +// What a client is sent when a turn failed. +// +// The three tiers are easy to collapse by accident — an edge that reaches for the protocol's +// default first will bury both an upstream's own words and a refusal's own envelope, and each +// family renders its own errors, so the rule is worth holding in one place and pinning here. + +import { describe, it } from 'vitest'; + +import { renderFailure } from '../../../src/data-plane/pipeline/facts.ts'; +import { assertEquals } from '@floway-dev/test-utils'; + +const protocolDefault = () => ({ error: { message: 'rendered from the status', type: 'api_error' } }); + +describe('renderFailure', () => { + it('forwards the body the upstream sent', () => { + const upstream = { error: { message: 'model is overloaded', type: 'overloaded_error' } }; + assertEquals( + renderFailure({ status: 529, message: 'upstream refused', body: upstream }, protocolDefault), + upstream, + ); + }); + + it('writes the refusal own envelope where the gateway refused', () => { + const envelope = { error: { message: 'no route', type: 'invalid_request_error', code: 'responses_item_routing_unavailable' } }; + assertEquals( + renderFailure({ status: 400, message: 'no route', envelope }, protocolDefault), + envelope, + ); + }); + + it('prefers what the upstream said over what the gateway would have written', () => { + const upstream = { error: { message: 'context length exceeded' } }; + const envelope = { error: { message: 'no route' } }; + assertEquals( + renderFailure({ status: 400, message: 'no route', body: upstream, envelope }, protocolDefault), + upstream, + ); + }); + + it('falls back to the protocol own rendering when neither is present', () => { + assertEquals(renderFailure({ status: 502, message: 'dial failed' }, protocolDefault), protocolDefault()); + }); + + it('renders the protocol default where the upstream body was not an object', () => { + // A gateway that forwarded `"Bad Gateway"` verbatim would answer a JSON protocol with a + // bare string, which no client of it can read. + assertEquals(renderFailure({ status: 502, message: 'dial failed', body: 'Bad Gateway' }, protocolDefault), protocolDefault()); + }); +}); diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 25d22da0c9..59eb0edc4f 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -26,7 +26,7 @@ import { billableUsageFromChatCompletionsEvent } from './usage.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; -import { isFailure } from '../../pipeline/facts.ts'; +import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -64,7 +64,7 @@ import { type ChatCompletionsPayload, type ChatCompletionsStreamEvent, } from '@floway-dev/protocols/chat-completions'; -import { renderProtocolError, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import type { BillableUsage, ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateChatCompletionsViaMessages, translateChatCompletionsViaResponses } from '@floway-dev/translate'; @@ -126,9 +126,9 @@ const emitChatCompletions = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.chatCompletions.rendered': move(renderProtocolError( - answer.body, - () => answer.envelope ?? { error: { message: answer.message, type: 'api_error' } }, + 'response.chat.chatCompletions.rendered': move(renderFailure( + answer, + () => ({ error: { message: answer.message, type: 'api_error' } }), )), 'response.http.status': answer.status, }; diff --git a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts index 9db3a81554..b9a58197fa 100644 --- a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts @@ -26,7 +26,7 @@ import { analyzeGeminiAffinity } from './affinity/ingress.ts'; import { renderGeminiError } from './errors.ts'; import type { GeminiFacts } from './pipeline.ts'; import { asJsonObject, readJsonNumber } from '../../../shared/json-helpers.ts'; -import { isFailure } from '../../pipeline/facts.ts'; +import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; @@ -39,7 +39,6 @@ import { messagesCountTokensWire, type TokenCountAnswer } from '../messages/coun import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; -import { renderProtocolError } from '@floway-dev/protocols/common'; import type { GeminiPayload } from '@floway-dev/protocols/gemini'; import { translateGeminiViaMessages } from '@floway-dev/translate'; @@ -85,7 +84,7 @@ const emitGeminiTokenCount = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(renderProtocolError(answer.body, () => renderGeminiError(answer.status, answer.message))), + 'response.chat.gemini.rendered': move(renderFailure(answer, () => renderGeminiError(answer.status, answer.message))), 'response.http.status': answer.status, }; } diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 34cfb21559..0ca220dbe8 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -47,7 +47,7 @@ import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; import { analyzeGeminiAffinity } from './affinity/ingress.ts'; import { renderGeminiError } from './errors.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; -import { isFailure } from '../../pipeline/facts.ts'; +import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -67,7 +67,7 @@ import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, transform, type Pipeline } from '@floway-dev/pipeline'; -import { renderProtocolError, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { collectGeminiProtocolEventsToResult, geminiProtocolFrameToSSEFrame, @@ -137,7 +137,7 @@ const emitGemini = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(renderProtocolError(answer.body, () => renderGeminiError(answer.status, answer.message))), + 'response.chat.gemini.rendered': move(renderFailure(answer, () => renderGeminiError(answer.status, answer.message))), 'response.http.status': answer.status, }; } diff --git a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts index 32642a7296..cd07e9f2a6 100644 --- a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts @@ -31,7 +31,7 @@ import { prepareMessagesWebSearchShimRequest } from './interceptors/web-search-s import type { MessagesFacts } from './pipeline.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; -import { isFailure } from '../../pipeline/facts.ts'; +import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; @@ -45,7 +45,6 @@ import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; -import { renderProtocolError } from '@floway-dev/protocols/common'; import { parseAnthropicBetaHeader, type MessagesPayload } from '@floway-dev/protocols/messages'; import { providerModelOf } from '@floway-dev/provider'; @@ -97,7 +96,7 @@ const emitMessagesTokenCount = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move(renderProtocolError(answer.body, () => renderMessagesError(answer.status, answer.message))), + 'response.chat.messages.rendered': move(renderFailure(answer, () => renderMessagesError(answer.status, answer.message))), 'response.http.status': answer.status, }; } diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index e3e13333e9..7aeef82854 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -24,7 +24,7 @@ import { createMessagesBillableUsageReader } from './usage.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; -import { isFailure } from '../../pipeline/facts.ts'; +import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -47,7 +47,7 @@ import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; -import { renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; +import { sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { collectMessagesProtocolEventsToResult, messagesProtocolFrameToSSEFrame, @@ -114,7 +114,7 @@ const emitMessages = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move(renderProtocolError(answer.body, () => renderMessagesError(answer.status, answer.message))), + 'response.chat.messages.rendered': move(renderFailure(answer, () => renderMessagesError(answer.status, answer.message))), 'response.http.status': answer.status, }; } diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index 77c6a72f12..727c586826 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -10,7 +10,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { Secret, Owned } from '@floway-dev/pipeline'; -import type { PricingRuntimeFacts } from '@floway-dev/protocols/common'; +import { renderProtocolError, type PricingRuntimeFacts } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; /** Everything about an attempt that is data: which upstream, which model row on it, and the @@ -55,6 +55,17 @@ export interface Failure { export const isFailure = (value: unknown): value is Failure => typeof value === 'object' && value !== null && 'status' in value && 'message' in value; +/** + * What a client is sent for a failure, in three tiers, so no edge has to remember them. + * + * An upstream that sent a body is forwarded it — the answer to "why was I refused" is the + * refusing party's to give. A refusal the gateway made itself is written by whoever made it, + * because only there is it known which field was at fault and which code names the condition. + * The protocol's own rendering of a status and a sentence is what is left when neither holds. + */ +export const renderFailure = (failure: Failure, protocolEnvelope: () => Record): Record => + renderProtocolError(failure.body, () => failure.envelope ?? protocolEnvelope()); + export interface GatewayFacts { /** What the client sent, before anything read it. Every family hands these over, because * every ending forwards what a provider is allowed to forward of them. */ From 08ab620b6e28c7cd4bc843344dac18e1ed9f03a1 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:00:36 +0800 Subject: [PATCH 056/156] fix(responses): answer a turn that asked for a compaction on the chain that serves it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compact shim ran on every Responses turn in the replaced surface, and only its compact half was carried onto the chain. So a `POST /v1/responses` whose input ended in a `compaction_trigger` was forwarded verbatim — to a translator that models no such item on a Messages or Chat Completions candidate — and a compaction this gateway had written reached an upstream as a blob it has no key for, losing the summarized conversation it stood for. Both halves are stages now, and both chains compose them: a compaction the shim wrote is issued through one entry and echoed back into the other, so neither half belongs to one of them. `expandShimCompactions` puts a blob of ours back into the items it encoded, and `summarizeForCompaction` — the stage the compaction chain already simulated with, now gated by what each chain reads as the ask — answers a `compaction_trigger` with an envelope of this gateway's own. Engagement is one reading, `simulatesCompaction`, taken by both endings and by both new stages: the operator's `responses-compact-shim` opt-in, or a candidate with no Responses endpoint at all. Where it does not hold, the trigger and the blob travel on to the upstream that answers them itself. --- .../chat/responses-pipeline_test.ts | 339 +++++++++++++++--- .../src/data-plane/chat/responses/compact.ts | 100 +----- .../src/data-plane/chat/responses/pipeline.ts | 187 +++++++++- 3 files changed, 493 insertions(+), 133 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index 152b8a2796..9a01ce58c8 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -4,20 +4,24 @@ // when the client did not ask to stream, that it hands the turn's own state back for the // client to carry, that the stored-items membrane's other half runs there too so the client // is answered under an envelope this gateway minted, that a turn the upstream answered with -// one envelope instead of a stream is served as that envelope, that a refusal keeps the -// upstream's own status and words, and that a dial nobody answered is a value the fork can -// move past. +// one envelope instead of a stream is served as that envelope, that a turn asking for a +// compaction is answered with one wherever the shim is this candidate's to run and travels on +// untouched where it is not, that a refusal keeps the upstream's own status and words, and +// that a dial nobody answered is a value the fork can move past. import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/responses/interceptors/compact-shim.ts'; import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import type { SseFrame } from '@floway-dev/protocols/common'; +import type { ModelEndpoints, SseFrame } from '@floway-dev/protocols/common'; +import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; import { RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type ResponsesCompactionResult, type ResponsesOutputItem, type ResponsesResult, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { directFetcher, type ModelCandidate, type ProviderResponsesResult } from '@floway-dev/provider'; +import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -28,18 +32,28 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal let live: readonly ModelCandidate[] = []; const candidate = ( - callResponses: (model: unknown, body: unknown, action: unknown) => Promise, - upstreamId = 'up_a', + calls: { + callResponses?: (model: unknown, body: unknown, action: unknown) => Promise; + callMessages?: (model: unknown, body: unknown) => Promise>; + }, + overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, ): ModelCandidate => { - const endpoints = { responses: {} }; + const upstreamId = overrides.upstreamId ?? 'up_a'; + const endpoints = overrides.endpoints ?? { responses: {} }; + const enabledFlags = overrides.enabledFlags ?? new Set(); return { provider: { upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callResponses }), + instance: stubProvider(calls as never), }, model: stubInternalModel( - { id: 'responses-model', endpoints, providerModels: { [upstreamId]: stubProviderModel({ id: 'responses-model', endpoints }) } }, + { + id: 'responses-model', + endpoints, + limits: { max_output_tokens: 4096 }, + providerModels: { [upstreamId]: stubProviderModel({ id: 'responses-model', endpoints, enabledFlags }) }, + }, upstreamId, ), fetcher: directFetcher, @@ -93,6 +107,34 @@ const stream = (...events: readonly ResponsesStreamEvent[]): ProviderResponsesRe })(), }); +/** One assistant message over the Messages wire, which is the only way a Messages-only + * candidate can be reached: the turn crosses into that protocol and comes back translated. */ +const messagesTurn = (text: string, seen: { body?: Record } = {}) => + async (_model: unknown, body: unknown): Promise> => { + seen.body = body as Record; + const events: unknown[] = [ + { + type: 'message_start', + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'responses-model', + stop_reason: null, stop_sequence: null, usage: { input_tokens: 12, output_tokens: 0 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 3 } }, + { type: 'message_stop' }, + ]; + return { + ok: true, modelKey: 'responses-model-key', headers: new Headers(), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event: event as MessagesStreamEvent }; + yield { type: 'done' as const }; + })(), + }; + }; + const payload = { model: 'responses-model', input: [{ type: 'message', role: 'user', content: 'hi' }], @@ -106,15 +148,20 @@ let affinityPayload: CanonicalResponsesPayload = payload; * anything below it reads the record. */ let asked = 0; -const serve = async (wantsStream: boolean) => await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream); +const serve = async (wantsStream: boolean, request: CanonicalResponsesPayload = payload) => + await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream, request); -const serveWith = async (gateway: ReturnType, wantsStream: boolean) => await run( - responsesServePipeline(payload), +const serveWith = async ( + gateway: ReturnType, + wantsStream: boolean, + request: CanonicalResponsesPayload = payload, +) => await run( + responsesServePipeline(request), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], 'ingress.chat.sourceProtocol': 'responses', 'ingress.chat.responses.wantsStream': wantsStream, - 'request.chat.responses': payload, + 'request.chat.responses': request, 'serve.model': 'responses-model', }) as never, { @@ -163,6 +210,37 @@ const withoutCarrier = (frames: readonly SseFrame[]): readonly SseFrame[] => return [{ ...frame, data: JSON.stringify({ ...event, response: withoutCarrierItem(event.response) }) }]; }); +/** A turn asking for a compaction, which on this protocol is an ordinary turn whose input + * ends in the control item that requests one. */ +const asksForCompaction = { + ...payload, + input: [ + { type: 'message', role: 'user', content: 'a long conversation' }, + { type: 'compaction_trigger' }, + ], +} as unknown as CanonicalResponsesPayload; + +/** The compaction item out of what the client was answered with. */ +const compactionItem = (answered: Record): { readonly encrypted_content: string } => { + const item = (answered.output as { type: string }[]).find(candidateItem => candidateItem.type === 'compaction'); + if (item === undefined) throw new Error('expected the answer to carry a compaction item'); + return item as unknown as { readonly encrypted_content: string }; +}; + +/** The summary a simulated compaction packed into the item the client is handed. The turn's + * own state is sealed around it on the way out — that is what pins the next turn to the + * upstream that issued this one — so the carrier is opened before the blob is read. */ +const summaryIn = async ( + gateway: ReturnType, + encryptedContent: string, +): Promise => { + const carrier = await gateway.affinity.codec.unwrap(encryptedContent, 'responses.compaction.encrypted_content'); + if (carrier.kind !== 'owned' || carrier.value === undefined) throw new Error('expected the turn-s own carrier around the blob'); + const items = decodeBase64UrlJson(carrier.value) as { content: { text: string }[] }[] | null; + if (items === null) throw new Error('expected a shim-encoded compaction blob'); + return items[0]!.content[0]!.text; +}; + beforeEach(() => { affinityPayload = payload; asked = 0; @@ -187,10 +265,12 @@ describe('the responses chain', () => { ...payload, input: [{ type: 'message', role: 'user', content: 'rewritten' }], } as CanonicalResponsesPayload; - resolves([candidate(async (_model, body, asks) => { - sent = body as Record; - action = asks; - return stream(completed('hi')); + resolves([candidate({ + callResponses: async (_model, body, asks) => { + sent = body as Record; + action = asks; + return stream(completed('hi')); + }, })]); const { facts } = await serve(false); @@ -218,9 +298,11 @@ describe('the responses chain', () => { { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'and then?' }] }, ], } as unknown as CanonicalResponsesPayload; - resolves([candidate(async (_model, body) => { - sent = body as Record; - return stream(completed('hi')); + resolves([candidate({ + callResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('hi')); + }, })]); await serve(false); @@ -237,7 +319,7 @@ describe('the responses chain', () => { // Every event goes out under its own SSE name, and the client's stream ends on the literal // `[DONE]` the transport reads as the turn being over. it('writes the frames out when the client asked to stream', async () => { - resolves([candidate(async () => stream(delta('he'), delta('llo'), completed('hello')))]); + resolves([candidate({ callResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); const { facts } = await serve(true); const frames = withoutCarrier(await drain(facts['response.chat.responses.rendered'])); @@ -273,7 +355,7 @@ describe('the responses chain', () => { // this is what the client is given to carry back. A turn that handed back nothing would // leave the follow-up quoting it with no way to name the upstream that issued it. it('hands the turn-s own state back on an item of its own', async () => { - resolves([candidate(async () => stream(delta('hi'), completed('hi')))]); + resolves([candidate({ callResponses: async () => stream(delta('hi'), completed('hi')) })]); const gateway = mockChatGatewayCtx({ wantsStream: true }); const { facts } = await serveWith(gateway, true); @@ -296,10 +378,12 @@ describe('the responses chain', () => { // A stream's numbers arrive with its last chunk, so what the run hands up is a promise and // the reading is taken off the frames the client itself drove. it('bills what the terminal event reported, once the frames have run out', async () => { - resolves([candidate(async () => stream( - delta('hi'), - completed('hi', { input_tokens: 11, output_tokens: 7, total_tokens: 18 }), - ))]); + resolves([candidate({ + callResponses: async () => stream( + delta('hi'), + completed('hi', { input_tokens: 11, output_tokens: 7, total_tokens: 18 }), + ), + })]); const { facts } = await serve(true); await drain(facts['response.chat.responses.rendered']); @@ -317,7 +401,7 @@ describe('the responses chain', () => { // Time to first token is measured where the token is — a lifecycle envelope is not one, and // a run that never stamps it is recorded as having produced nothing to time. it('stamps the first frame that carried a generated token', async () => { - resolves([candidate(async () => stream(delta('hi'), completed('hi')))]); + resolves([candidate({ callResponses: async () => stream(delta('hi'), completed('hi')) })]); const gateway = mockChatGatewayCtx({ wantsStream: true }); const { facts } = await serveWith(gateway, true); @@ -331,7 +415,7 @@ describe('the responses chain', () => { // of the answer and the client is not shown it. Reading stops there, which also closes the // upstream rather than leaving the client's stream open behind a connection nobody reads. it('stops reading at the terminal event', async () => { - resolves([candidate(async () => stream(delta('hi'), completed('hi'), delta(' and more')))]); + resolves([candidate({ callResponses: async () => stream(delta('hi'), completed('hi'), delta(' and more')) })]); const { facts } = await serve(true); const frames = withoutCarrier(await drain(facts['response.chat.responses.rendered'])); @@ -350,7 +434,7 @@ describe('the responses chain', () => { // words — and the stream ends on that rather than on the terminator, because a stream that // ended on `[DONE]` is a stream that finished. it('tells a client already being streamed to that the stream never ended', async () => { - resolves([candidate(async () => stream(delta('hi')))]); + resolves([candidate({ callResponses: async () => stream(delta('hi')) })]); const { facts } = await serve(true); const frames = await drain(facts['response.chat.responses.rendered']); @@ -368,7 +452,7 @@ describe('the responses chain', () => { // The upstream speaks SSE whatever the client asked for, so a client that did not ask to // stream is answered from the same frames — folded here rather than read a second time. it('folds the frames into one response when the client did not', async () => { - resolves([candidate(async () => stream(delta('he'), delta('llo'), completed('hello')))]); + resolves([candidate({ callResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); const { facts } = await serve(false); @@ -395,9 +479,11 @@ describe('the responses chain', () => { }], usage: { input_tokens: 900, output_tokens: 40, total_tokens: 940 }, }; - resolves([candidate(async () => ({ - action: 'compact', ok: true, modelKey: 'responses-model-key', result: compaction, - }))]); + resolves([candidate({ + callResponses: async () => ({ + action: 'compact', ok: true, modelKey: 'responses-model-key', result: compaction, + }), + })]); const { facts } = await serve(false); @@ -412,7 +498,7 @@ describe('the responses chain', () => { // Content-length would misdescribe a body this gateway serialized itself; a vendor trace // is what a client and an operator both need to correlate a turn. it('forwards the upstream headers a client may see and drops the ones it may not', async () => { - resolves([candidate(async () => stream(completed('hi')))]); + resolves([candidate({ callResponses: async () => stream(completed('hi')) })]); const { facts } = await serve(true); @@ -421,14 +507,16 @@ describe('the responses chain', () => { }); it('answers an upstream refusal with its own status and words', async () => { - resolves([candidate(async () => ({ - action: 'generate', - ok: false, - modelKey: 'responses-model-key', - response: new Response(JSON.stringify({ error: { message: 'slow down' } }), { - status: 429, headers: { 'content-type': 'application/json' }, + resolves([candidate({ + callResponses: async () => ({ + action: 'generate', + ok: false, + modelKey: 'responses-model-key', + response: new Response(JSON.stringify({ error: { message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), }), - }))]); + })]); const { facts } = await serve(false); @@ -441,8 +529,8 @@ describe('the responses chain', () => { it('fails a dial that never connected over to the next candidate', async () => { const tried: string[] = []; resolves([ - candidate(async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); }, 'up_dead'), - candidate(async () => { tried.push('alive'); return stream(completed('hi')); }, 'up_alive'), + candidate({ callResponses: async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); } }, { upstreamId: 'up_dead' }), + candidate({ callResponses: async () => { tried.push('alive'); return stream(completed('hi')); } }, { upstreamId: 'up_alive' }), ]); const { facts } = await serve(false); @@ -450,4 +538,167 @@ describe('the responses chain', () => { expect(tried).toEqual(['dead', 'alive']); expect(facts['response.http.status']).toBe(200); }); + + // Codex asks for a compaction inside an ordinary turn, by ending its input with the control + // item that requests one. Where the shim is this candidate's to run, that item never reaches + // the upstream: what is sent instead is the compactor's own turn, and what comes back is an + // envelope this gateway packed the summary into. + it('answers a turn that asked for a compaction with one, where the shim is engaged', async () => { + let sent: Record | undefined; + affinityPayload = asksForCompaction; + resolves([candidate( + { + callResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('CONDENSED SUMMARY')); + }, + }, + { enabledFlags: new Set(['responses-compact-shim']) }, + )]); + const gateway = mockChatGatewayCtx({ wantsStream: false }); + + const { facts } = await serveWith(gateway, false, asksForCompaction); + + // The compactor's prompt reached the upstream, the item that asked for the compaction did + // not, and the ephemeral summarization turn is not persisted in the upstream's history. + expect(JSON.stringify(sent)).toContain('CONTEXT CHECKPOINT COMPACTION'); + expect(JSON.stringify(sent)).not.toContain('compaction_trigger'); + expect(sent).toMatchObject({ store: false }); + + expect(facts['response.http.status']).toBe(200); + const answered = facts['response.chat.responses.rendered'] as Record; + expect(answered.object).toBe('response.compaction'); + expect(await summaryIn(gateway, compactionItem(answered).encrypted_content)).toBe(`${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); + }); + + // The flag says where a compaction would be simulated, not that this turn asked for one — + // so an ordinary turn on an opted-in upstream is dialled as itself and answered as a + // response, with nothing of the compactor's in the body that went out. + it('dials an ordinary turn as itself, where the shim is engaged but nothing asked for one', async () => { + let sent: Record | undefined; + resolves([candidate( + { + callResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('hello')); + }, + }, + { enabledFlags: new Set(['responses-compact-shim']) }, + )]); + + const { facts } = await serve(false); + + expect(sent).toMatchObject({ input: [{ type: 'message', role: 'user', content: 'hi' }] }); + expect(JSON.stringify(sent)).not.toContain('CONTEXT CHECKPOINT COMPACTION'); + const answered = withoutCarrierItem(facts['response.chat.responses.rendered'] as unknown as ResponsesResult); + expect(answered.object).toBe('response'); + expect(answered).toMatchObject({ output: [{ content: [{ type: 'output_text', text: 'hello' }] }] }); + }); + + // The flag is the operator's opt-in, so an upstream that did not get it answers the item + // itself: it reaches the wire as the client wrote it, and the compaction that comes back is + // the upstream's own — its id, and a blob only it can read. + it('sends a turn that asked for a compaction on unchanged, where the shim is not engaged', async () => { + let sent: Record | undefined; + const upstreamCompaction: ResponsesCompactionResult = { + id: 'resp_upstream_compaction', + object: 'response.compaction', + output: [{ type: 'compaction', id: 'cmp_1', encrypted_content: 'OPAQUE_NATIVE_BLOB' } as unknown as never], + usage: { input_tokens: 900, output_tokens: 40, total_tokens: 940 }, + }; + affinityPayload = asksForCompaction; + resolves([candidate({ + callResponses: async (_model, body) => { + sent = body as Record; + return { action: 'compact', ok: true, modelKey: 'responses-model-key', result: upstreamCompaction }; + }, + })]); + + const { facts } = await serve(false, asksForCompaction); + + expect(sent).toMatchObject({ input: [{ role: 'user' }, { type: 'compaction_trigger' }] }); + expect(JSON.stringify(sent)).not.toContain('CONTEXT CHECKPOINT COMPACTION'); + expect(facts['response.chat.responses.rendered']).toEqual(upstreamCompaction); + }); + + // No translation carries a compaction, and neither translator models the item that asks for + // one — so on a candidate this protocol can only reach through one, the shim is structurally + // required rather than opted into, and the flag has nothing to say. + it('answers a turn that asked for a compaction over a candidate with no Responses wire', async () => { + const seen: { body?: Record } = {}; + affinityPayload = asksForCompaction; + resolves([candidate( + { callMessages: messagesTurn('CONDENSED SUMMARY', seen) }, + { endpoints: { messages: {} } }, + )]); + const gateway = mockChatGatewayCtx({ wantsStream: false }); + + const { facts } = await serveWith(gateway, false, asksForCompaction); + + expect(JSON.stringify(seen.body)).toContain('CONTEXT CHECKPOINT COMPACTION'); + expect(JSON.stringify(seen.body)).not.toContain('compaction_trigger'); + const answered = facts['response.chat.responses.rendered'] as Record; + expect(answered.object).toBe('response.compaction'); + expect(await summaryIn(gateway, compactionItem(answered).encrypted_content)).toBe(`${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); + }); + + // A compaction this gateway simulated carries the history it stood for, so an ordinary turn + // that echoes one back is sent that history — the upstream has no key for the blob, and + // would otherwise continue from nothing. + it('expands a compaction it wrote before the turn quoting it goes out', async () => { + let sent: Record | undefined; + const continues = { + ...payload, + input: [ + { + type: 'compaction', + id: 'cmp_prior', + encrypted_content: encodeBase64UrlJson([ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'THE EARLIER HISTORY' }] }, + ]), + }, + { type: 'message', role: 'user', content: 'and then?' }, + ], + } as unknown as CanonicalResponsesPayload; + affinityPayload = continues; + resolves([candidate( + { + callResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('hi')); + }, + }, + { enabledFlags: new Set(['responses-compact-shim']) }, + )]); + + await serve(false, continues); + + expect(JSON.stringify(sent)).toContain('THE EARLIER HISTORY'); + expect(JSON.stringify(sent)).not.toContain('cmp_prior'); + }); + + // Only an upstream whose compactions this gateway simulates can be holding one of ours, and + // an upstream that compacts natively is owed its own blob byte for byte — so where the shim + // is not engaged the item travels as the client wrote it. + it('leaves a compaction alone where the shim is not engaged', async () => { + let sent: Record | undefined; + const encoded = encodeBase64UrlJson([ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'THE EARLIER HISTORY' }] }, + ]); + const continues = { + ...payload, + input: [{ type: 'compaction', id: 'cmp_prior', encrypted_content: encoded }], + } as unknown as CanonicalResponsesPayload; + affinityPayload = continues; + resolves([candidate({ + callResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('hi')); + }, + })]); + + await serve(false, continues); + + expect(sent).toMatchObject({ input: [{ type: 'compaction', id: 'cmp_prior', encrypted_content: encoded }] }); + }); }); diff --git a/packages/gateway/src/data-plane/chat/responses/compact.ts b/packages/gateway/src/data-plane/chat/responses/compact.ts index 910848f234..92bb3dc5dc 100644 --- a/packages/gateway/src/data-plane/chat/responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/responses/compact.ts @@ -7,6 +7,7 @@ // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record // beginStoredAttempt reseeds the store's per-attempt scratchpad +// expandShimCompactions a compaction this gateway wrote, back into what it stood for // this family's request rules, the same four generation runs above its fork // dialResponsesCompaction the ending: dials a compaction, or simulates one // @@ -14,7 +15,8 @@ // `responsesServePipeline`, which is why it is a chain of its own. Everything above the // ending it shares with generation, stage for stage, because a compaction is routed, pinned // and rewritten exactly as a turn is: the same membrane hydrates it, the same narrowing -// orders its candidates, the same four rules shape the body that goes out. +// orders its candidates, the same rules shape the body that goes out — the shim's own +// expansion among them, because a compaction it wrote is echoed back into either entry. // // The ending is where the two part. Two wires, both handing up `response.chat.responses`: // @@ -26,8 +28,10 @@ // `responses-compact-shim` — is sent an ordinary generate turn against the compactor's // prompt over the wires generation would have used, and the summary it produces is packed // into an envelope of this gateway's own. That is the whole of what the action pivot was: -// one wire dials `compact`, the other dials `generate` under a stage that folds the -// answer, and neither needs an action to travel in the record. +// one wire dials `compact`, the other dials `generate` under the stage that folds the +// answer, and neither needs an action to travel in the record. The stage is the one the +// generate chain answers a `compaction_trigger` with, so whichever entry asked, the +// compaction that comes back was made the same way. // // What the simulation cannot reproduce is stated where it happens, at `summarizationTurnFor`. // @@ -38,17 +42,11 @@ import { responsesCreatedAt, wrapResponsesStatefulOutput } from './client-output.ts'; import { completeResponsesCompaction } from './compaction-resource.ts'; -import { - buildCompactionEnvelope, - EMPTY_SUMMARY_MESSAGE, - expandShimCompactionItems, - summarizationTurnFor, - summaryTextFrom, -} from './interceptors/compact-shim.ts'; -import { syntheticEventsFromCompaction, syntheticEventsFromResult } from './items/output.ts'; +import { syntheticEventsFromCompaction } from './items/output.ts'; import { beginStoredAttempt, billedResponsesEntity, + expandShimCompactions, hydrateStoredItems, internalErrorEnvelope, RESPONSES_STREAMED_USAGE, @@ -56,6 +54,8 @@ import { responsesTarget, responsesWireFor, responsesWireRules, + simulatesCompaction, + summarizeForCompaction, type ResponsesFacts, } from './pipeline.ts'; import { billableUsageFromResponsesResult } from './usage.ts'; @@ -80,9 +80,7 @@ import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline' import { renderProtocolError, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, - createRandomResponsesItemId, type CanonicalResponsesPayload, - type ResponsesOutputItem, type ResponsesStreamEvent, } from '@floway-dev/protocols/responses'; import { providerModelOf } from '@floway-dev/provider'; @@ -292,74 +290,6 @@ const callResponsesCompactUpstream = defineStage< }, }); -/** - * Simulates a compaction over an upstream that has no compaction wire. - * - * It rewrites the turn on the way down — the compactor's prompt at the head of the history, - * the trigger stripped, a terminal nudge appended, nothing persisted upstream — and folds the - * generated summary into an envelope of this gateway's own on the way back. Below it is an - * ordinary generate fork, which is what makes every wire generation can take a wire a - * compaction can be simulated over. - * - * A prior simulated compaction the client echoed back is expanded first, so what is summarized - * is the history it stood for rather than an opaque blob. A blob this gateway did not write - * round-trips untouched. - */ -const summarizeForCompaction = defineStage< - R<'request.chat.responses'>, - R<'request.chat.responses'>, - Compacted<'response.chat.responses'>, - Compacted<'response.chat.responses'>, - ChatServices ->({ - name: 'summarizeForCompaction', - through: { - request: { needs: ['request.chat.responses'], consumes: [], provides: ['request.chat.responses'] }, - response: { - needs: ['response.chat.responses'], - consumes: [], - provides: ['response.chat.responses'], - }, - }, - execute: async (facts, next) => { - const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; - const back = await next({ - ...facts, - 'request.chat.responses': move(summarizationTurnFor(expandShimCompactionItems(asked))), - }); - - const answer = back['response.chat.responses']; - // An upstream that refused is handed on as it came: the client learns the compaction - // failed rather than being given a silent empty envelope. - if (isFailure(answer)) return back; - - // The item lifecycle is the authority on what the turn closed; a Codex upstream states an - // `output` on its terminal that omits the message it just closed, so both are read and - // the closed items win where there are any. - const closed = new Map(); - const observed = (async function* (): AsyncIterable> { - for await (const frame of answer.frames as AsyncIterable>) { - if (frame.type === 'event' && frame.event.type === 'response.output_item.done') { - closed.set(frame.event.output_index, frame.event.item); - } - yield frame; - } - })(); - const collected = await collectResponsesProtocolEventsToResult(observed); - - const summaryText = summaryTextFrom(closed, collected.output); - if (summaryText.length === 0) { - // A summarization that closed no text produced no summary, and the blob is the whole of - // what the next turn inherits — so this is a candidate that did not do the job rather - // than a fault that ends the request, and the fork can try another. - return { ...back, 'response.chat.responses': move({ status: 502, message: EMPTY_SUMMARY_MESSAGE }) }; - } - - const synthesized = buildCompactionEnvelope(createRandomResponsesItemId('compaction'), summaryText, collected); - return { ...back, 'response.chat.responses': move({ kind: 'stream' as const, frames: syntheticEventsFromResult(synthesized) }) }; - }, -}); - /** The generate fork, as the simulation reaches it. A summarization is an ordinary turn, so * it is dialled on the wires an ordinary turn is dialled on. */ const dialSummarizationWire = dialChatWire({ @@ -371,7 +301,9 @@ const dialSummarizationWire = dialChatWire({ }); const compactionWire: ChatWire = compose('responsesCompactNative', [...responsesWireRules, callResponsesCompactUpstream]); -const simulationWire: ChatWire = compose('responsesCompactSimulated', [summarizeForCompaction, dialSummarizationWire]); +// The ending below picks this wire only for a candidate whose compactions are the shim's to +// simulate, so the ask is already answered: every turn that reaches it is one to summarize. +const simulationWire: ChatWire = compose('responsesCompactSimulated', [summarizeForCompaction(() => true), dialSummarizationWire]); /** * Picks how this candidate's compaction is produced. @@ -415,8 +347,7 @@ const dialResponsesCompaction = defineStage< }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - const compacts = responsesTarget.pick(candidate.model.endpoints) === 'responses' - && !facts['route.attempt'].flags.includes('responses-compact-shim'); + const compacts = !simulatesCompaction(candidate, facts['route.attempt']); use.log.debug('compacting', { upstream: facts['route.attempt'].upstreamId, wire: compacts ? 'compact' : 'simulated' }); return await next(facts, compacts ? compactionWire : simulationWire); }, @@ -450,6 +381,7 @@ export const responsesCompactPipeline = (payload: CanonicalResponsesPayload): Pi }), materializeAttempt('request.chat.responses'), beginStoredAttempt, + expandShimCompactions, disableReasoningOnForcedToolChoiceForResponses, stripPromptCacheKeyForResponses, vendorDeepSeekNormalizeForResponses, diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index 1cef4cf152..a10d819dfc 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -7,6 +7,8 @@ // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record // beginStoredAttempt reseeds the store's per-attempt scratchpad +// expandShimCompactions a compaction this gateway wrote, back into what it stood for +// summarizeForCompaction answers a turn that asked for a compaction with one // dialChatWire the ending: picks this candidate's wire and hands into it // // Three wires, all handing up `response.chat.responses`: this protocol's own, and the two @@ -26,10 +28,10 @@ // // - `/v1/responses/compact`. A second operation over this protocol rather than another // wire under this pipeline, so it is a chain of its own in `compact.ts` — which reuses -// this one's membrane, its narrowing and its wires, because a compaction routes and is -// rewritten exactly as a turn is. The dial here asks for `generate`; what the ending -// answers with is the branch the provider says it ran, which is why the envelope a -// compaction is arrives somewhere rather than nowhere. +// this one's membrane, its narrowing, its wires and the two shim rules below, because a +// compaction routes and is rewritten exactly as a turn is. The dial here asks for +// `generate`; what the ending answers with is the branch the provider says it ran, which +// is why the envelope a compaction is arrives somewhere rather than nowhere. // - this family's remaining interceptors, the server-tool shims among them. Still only in // the interceptor form, so the array between the materialized payload and the fork is // short rather than complete — and nothing in a pipelined turn writes to the store's @@ -42,11 +44,20 @@ import { analyzeResponsesAffinity } from './affinity/ingress.ts'; import { wrapResponsesClientEgress } from './client-output.ts'; import type { ResponsesServeFailure } from './errors.ts'; +import { + buildCompactionEnvelope, + containsCompactionTrigger, + EMPTY_SUMMARY_MESSAGE, + expandShimCompactionItems, + summarizationTurnFor, + summaryTextFrom, +} from './interceptors/compact-shim.ts'; import { hydrateResponsesPayload } from './items/hydrate.ts'; import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; +import { syntheticEventsFromResult } from './items/output.ts'; import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve-prep.ts'; import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; -import type { BillableEntity } from '../../pipeline/facts.ts'; +import type { AttemptSelector, BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; @@ -77,12 +88,14 @@ import { compose, defineStage, move, type Pipeline, type Stage, type Use } from import { doneFrame, eventFrame, renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, + createRandomResponsesItemId, isResponsesTerminalEvent, responsesProtocolFrameToSSEFrame, RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type ClientResponseResource, type ClientResponsesStreamEvent, + type ResponsesOutputItem, type ResponsesStreamEvent, } from '@floway-dev/protocols/responses'; import { providerModelOf, toInternalDebugError, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; @@ -763,6 +776,168 @@ export const beginStoredAttempt = defineStage< }, }); +// ── The compact shim ────────────────────────────────────────────────────────────────────── +// +// Two rules, composed by both of this protocol's chains, because a compaction the shim wrote +// is issued through one entry and read back through the other: Codex asks for one by ending +// a generate turn's input with a `compaction_trigger`, `/v1/responses/compact` asks for one +// by being called at all, and either way the envelope comes back to the client, who echoes it +// into the ordinary turns that follow. What the shim is, what it vendors and what it cannot +// reproduce is at `interceptors/compact-shim.ts`. + +/** + * Whether this candidate's compactions are the shim's to simulate. + * + * Two ways to be, and an upstream only has to be one of them. The operator opted a Responses + * upstream in with `responses-compact-shim`, which is the say-so for one that would answer a + * compaction itself. Or the candidate has no Responses endpoint at all: no translation + * carries a compaction, so a Messages or Chat Completions candidate has neither a compaction + * to dial nor a translator that models the item asking for one — the shim is structurally + * required there rather than chosen. + * + * One reading, taken wherever the question is asked, so a turn that asks for a compaction on + * its way through generation is answered with the compaction this protocol's own endpoint + * would have produced for it. + */ +export const simulatesCompaction = (candidate: ModelCandidate, attempt: AttemptSelector): boolean => + responsesTarget.pick(candidate.model.endpoints) !== 'responses' + || attempt.flags.includes('responses-compact-shim'); + +/** + * Expands a compaction this gateway wrote back into the history it stood for. + * + * The shim's envelope carries the items it summarized, so a turn that echoes one back is sent + * that history rather than an opaque blob the upstream has no key for. A blob this gateway + * did not write — a native upstream's own encrypted compaction — fails the decode and rides + * through untouched, which is what lets an operator turn the flag off for an upstream that + * compacts natively. + * + * Gated on the same reading the endings take: only an upstream whose compactions this gateway + * simulates can be holding one of ours, and a native one is owed its own blob verbatim. + */ +export const expandShimCompactions = defineStage< + R<'request.chat.responses' | 'route.attempt'>, + R<'request.chat.responses'>, + Record, + Record, + ChatServices +>({ + name: 'expandShimCompactions', + through: { + request: { + needs: ['request.chat.responses', 'route.attempt'], + consumes: [], + provides: ['request.chat.responses'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => { + if (!simulatesCompaction(use.resolveAttempt(facts['route.attempt']), facts['route.attempt'])) { + return await next(facts); + } + // The key holds what a client may send, whose `input` is a string or a list; this chain + // runs on the canonical form the entry normalized it to. + const payload = facts['request.chat.responses'] as CanonicalResponsesPayload; + const expanded = expandShimCompactionItems(payload); + // A turn carrying none of ours hands the same payload on, so the record shows no change + // where none happened. + if (expanded === payload) return await next(facts); + return await next({ ...facts, 'request.chat.responses': move(expanded) }); + }, +}); + +/** How the chain that composed the stage below says a turn reaching it asked for a compaction. + * It is the whole gate, so each chain states its own: what asks for a compaction differs by + * operation, and no action travels in the record for the stage to read instead. */ +export type CompactionAsk = ( + facts: R<'request.chat.responses' | 'route.attempt'>, + use: Use, +) => boolean; + +/** + * Answers a turn that asked for a compaction with one, over an upstream that has no + * compaction wire. + * + * It rewrites the turn on the way down — the compactor's prompt at the head of the history, + * the trigger stripped, a terminal nudge appended, nothing persisted upstream — and folds the + * generated summary into an envelope of this gateway's own on the way back. Below it is the + * ordinary generate fork, which is what makes every wire generation can take a wire a + * compaction can be simulated over. + * + * What the simulation cannot reproduce is stated where it happens, at `summarizationTurnFor`. + */ +export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< + R<'request.chat.responses' | 'route.attempt'>, + R<'request.chat.responses'>, + R<'response.chat.responses'>, + R<'response.chat.responses'>, + ChatServices +>({ + name: 'summarizeForCompaction', + through: { + request: { + needs: ['request.chat.responses', 'route.attempt'], + consumes: [], + provides: ['request.chat.responses'], + }, + response: { + needs: ['response.chat.responses'], + consumes: [], + provides: ['response.chat.responses'], + }, + }, + execute: async (facts, next, use) => { + if (!asked(facts, use)) return await next(facts); + const payload = facts['request.chat.responses'] as CanonicalResponsesPayload; + const back = await next({ ...facts, 'request.chat.responses': move(summarizationTurnFor(payload)) }); + + const answer = back['response.chat.responses']; + // An upstream that refused is handed on as it came: the client learns the compaction + // failed rather than being given a silent empty envelope. One that answered with a single + // envelope compacted on its own, and an envelope is already the answer — there are no + // frames to fold and nothing this layer could add to it. + if (isFailure(answer) || answer.kind !== 'stream') return back; + + // The item lifecycle is the authority on what the turn closed; a Codex upstream states an + // `output` on its terminal that omits the message it just closed, so both are read and + // the closed items win where there are any. + const closed = new Map(); + const observed = (async function* (): AsyncIterable> { + for await (const frame of answer.frames as AsyncIterable>) { + if (frame.type === 'event' && frame.event.type === 'response.output_item.done') { + closed.set(frame.event.output_index, frame.event.item); + } + yield frame; + } + })(); + const collected = await collectResponsesProtocolEventsToResult(observed); + + const summaryText = summaryTextFrom(closed, collected.output); + if (summaryText.length === 0) { + // A summarization that closed no text produced no summary, and the blob is the whole of + // what the next turn inherits — so this is a candidate that did not do the job rather + // than a fault that ends the request, and the fork can try another. + return { ...back, 'response.chat.responses': move({ status: 502, message: EMPTY_SUMMARY_MESSAGE }) }; + } + + const synthesized = buildCompactionEnvelope(createRandomResponsesItemId('compaction'), summaryText, collected); + return { ...back, 'response.chat.responses': move({ kind: 'stream' as const, frames: syntheticEventsFromResult(synthesized) }) }; + }, +}); + +/** + * What asking for a compaction looks like on this chain. + * + * Codex's RemoteCompactionV2 path asks for one inside an ordinary turn, by ending its input + * with the control item that requests it — so this chain reads the request rather than the + * operation, and reads it per turn. Where the shim is not this candidate's to run, the item + * travels on to an upstream whose own `/responses` endpoint answers it, which is what a + * native Responses upstream does with it. + */ +const asksForCompaction: CompactionAsk = (facts, use) => + containsCompactionTrigger((facts['request.chat.responses'] as CanonicalResponsesPayload).input) + && simulatesCompaction(use.resolveAttempt(facts['route.attempt']), facts['route.attempt']); + export type ResponsesServeEntry = R< 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.responses.wantsStream' | 'request.chat.responses' | 'serve.model' @@ -798,6 +973,8 @@ export const responsesServePipeline = ( }), materializeAttempt('request.chat.responses'), beginStoredAttempt, + expandShimCompactions, + summarizeForCompaction(asksForCompaction), dialChatWire({ source: 'request.chat.responses', needs: ['request.chat.responses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], From 5bff74e56e6cf4b215ad26777d8d767d3c578fc1 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:07:49 +0800 Subject: [PATCH 057/156] refactor(responses): delete the compact shim's interceptor form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The onion's removal kept this file for "the compaction shim's summarization" — what a chain still reaches — but `withResponsesCompactShim` came along with it and has had no caller since. Now that both of its halves are stages, keeping it would leave the engagement rule and the envelope fold stated twice, free to drift from the one the chains run. What stays is the substance: the vendored compactor prompt and handoff prefix with their extraction evidence, the inbound expansion, the summarization turn, the summary reading and the envelope. The file's header says where the rules that use them run. Its tests follow the same split the rest of the migration took: the orchestration is checked where it happens, on the two chains, and what is written down here is the shape of the turn the compactor is sent and of the envelope it comes back in. --- .../interceptors/compact-shim_test.ts | 645 ++++-------------- .../responses/interceptors/compact-shim.ts | 150 +--- 2 files changed, 181 insertions(+), 614 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/compact-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/compact-shim_test.ts index 01c2508661..82e9e83d09 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/compact-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/compact-shim_test.ts @@ -1,54 +1,53 @@ +// What a simulated compaction is made of. The rules that use these run as stages and are +// tested where they run — `responses-pipeline_test.ts` for the generate chain and +// `responses-compact_test.ts` for the compaction chain; what is written down here is the +// substance those stages send and pack. + import { test } from 'vitest'; -import { SUMMARY_PREFIX, expandShimCompactionItems, withResponsesCompactShim } from '../../../../../src/data-plane/chat/responses/interceptors/compact-shim.ts'; -import type { ResponsesInvocation } from '../../../../../src/data-plane/chat/responses/interceptors/types.ts'; -import { encodeBase64UrlJson } from '../../../../../src/shared/base64url-json.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectResponsesProtocolEventsToResult, type CanonicalResponsesPayload, type ResponsesInputItem, type ResponsesPayload, type ResponsesResult, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { eventResult, type ExecuteResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const makeInvocation = ( - payload: Partial = {}, - options: { action?: 'generate' | 'compact'; flagOn?: boolean; targetApi?: 'responses' | 'messages' | 'chat-completions' } = {}, -): ResponsesInvocation => ({ - payload: { model: 'test-model', input: [], ...payload } as CanonicalResponsesPayload, - action: options.action ?? 'generate', - candidate: stubModelCandidate({ enabledFlags: new Set(options.flagOn === false ? [] : ['responses-compact-shim']) }), - targetApi: options.targetApi ?? 'responses', - headers: new Headers(), +import { + buildCompactionEnvelope, + expandShimCompactionItems, + summarizationTurnFor, + summaryTextFrom, + SUMMARY_PREFIX, +} from '../../../../../src/data-plane/chat/responses/interceptors/compact-shim.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../../src/shared/base64url-json.ts'; +import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesOutputItem, ResponsesResult } from '@floway-dev/protocols/responses'; +import { assertEquals } from '@floway-dev/test-utils'; + +const turnFor = (payload: Partial = {}): CanonicalResponsesPayload => + summarizationTurnFor({ model: 'test-model', input: [], ...payload } as CanonicalResponsesPayload); + +/** The items a turn was rewritten into, as the assertions below read them. */ +const itemsOf = (payload: CanonicalResponsesPayload): { type: string; role?: string; content?: { type: string; text: string }[] }[] => + payload.input as { type: string; role?: string; content?: { type: string; text: string }[] }[]; + +/** What the summarization turn produced, as an upstream states it. */ +const summarized = (text: string, overrides: Partial = {}): ResponsesResult => ({ + id: 'resp_fake_upstream', + object: 'response', + model: 'test-upstream-model', + status: 'completed', + output: [{ + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text, annotations: [] }], + }], + error: null, + incomplete_details: null, + usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, + ...overrides, }); -// Build a fake upstream `run()` that emits a single completed response whose -// output contains one assistant message with the given text. Used to model -// the inner summarization turn the shim drives. -const fakeUpstreamRun = (summaryText: string): () => Promise>> => { - const response: ResponsesResult = { - id: 'resp_fake_upstream', - object: 'response', - model: 'test-upstream-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: summaryText, annotations: [] }], - }], - error: null, - incomplete_details: null, - usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, - }; - return () => Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.completed', sequence_number: 0, response }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - )); +/** The items a compaction blob stands for. The blob is this gateway's own base64url-JSON + * marker, so what a test reads back is what a later turn's expansion would. */ +const decodeBlob = (encryptedContent: string): { type: string; role: string; content: { type: string; text: string }[] }[] => { + const decoded = decodeBase64UrlJson(encryptedContent); + if (decoded === null) throw new Error('expected a shim-encoded compaction blob'); + return decoded as { type: string; role: string; content: { type: string; text: string }[] }[]; }; // ── Inbound expansion (expandShimCompactionItems) ──────────────────────────── @@ -97,49 +96,21 @@ test('inbound: foreign compaction blob (valid base64url but wrong shape) round-t assertEquals(expanded, original); }); -// ── Outbound summarization (withResponsesCompactShim) ──────────────────────── - -test('compact + flag on: pivots to generate, drives upstream summarization, returns compaction envelope', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'long conversation history' }] }, - { action: 'compact' }, - ); +// ── The summarization turn (summarizationTurnFor) ──────────────────────────── - let seenPayload: ResponsesPayload | undefined; - let seenAction: 'generate' | 'compact' | undefined; - const result = await withResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - seenAction = inv.action; - return fakeUpstreamRun('CONDENSED SUMMARY')(); - }); +test('the turn carries SUMMARIZATION_PROMPT as a role=system item at the head of the history, and persists nothing', () => { + const turn = turnFor({ input: [{ type: 'message', role: 'user', content: 'long conversation history' }] }); - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - // Inner action seen by the upstream is 'generate'. - assertEquals(seenAction, 'generate'); - // The pivot is one-way per the project's interceptor convention: outer - // ctx.action stays 'generate' after the run. attempt.invoke keys - // envelope-drain on the caller's intent action (captured by value), so - // leaving invocation.action='generate' does not change the result shape. - assertEquals(inv.action, 'generate'); - // Payload pivoted: SUMMARIZATION_PROMPT injected as a role=system input - // item at the head of the history (mirroring native compact's system-role - // compactor prompt), store:false, the original history retained after it - // (compaction_trigger items would be stripped but there are none here). - if (!seenPayload) throw new Error('expected the upstream call to see the rewritten payload'); - const rewrittenInput = seenPayload.input as ResponsesInputItem[]; - const compactorSystem = rewrittenInput[0] as { type: string; role: string; content: Array<{ text: string }> }; + const compactorSystem = itemsOf(turn)[0]!; assertEquals(compactorSystem.type, 'message'); assertEquals(compactorSystem.role, 'system'); - assertEquals(compactorSystem.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); - assertEquals(seenPayload.store, false); - - const collected = await collectResponsesProtocolEventsToResult(result.events); - assertEquals(collected.object, 'response.compaction'); - const compactionItem = collected.output[0] as { type: string; encrypted_content: string }; - assertEquals(compactionItem.type, 'compaction'); + assertEquals(compactorSystem.content![0]!.text.includes('CONTEXT CHECKPOINT COMPACTION'), true); + // The ephemeral summarization turn does not pollute the upstream's own conversation + // history. + assertEquals(turn.store, false); }); -test("compact + flag on: caller's `instructions` field flows through untouched (bug-for-bug parity with native compact)", async () => { +test("the caller's `instructions` field flows through untouched (bug-for-bug parity with native compact)", () => { // Native `/responses/compact` puts SUMMARIZATION_PROMPT into the compactor // context as a role=system message AND forwards the caller's `instructions` // as a developer-role message alongside; both are in scope. Prompt- @@ -149,463 +120,149 @@ test("compact + flag on: caller's `instructions` field flows through untouched ( // summary. The shim must reproduce that shape — do NOT overwrite the // caller's instructions with SUMMARIZATION_PROMPT (it now lives as a // separate role=system input item). - const inv = makeInvocation( - { - input: [{ type: 'message', role: 'user', content: 'history' }], - instructions: 'You are a helpful assistant. Always mention the word quokka.', - }, - { action: 'compact' }, - ); - - let seenPayload: ResponsesPayload | undefined; - await withResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('irrelevant')(); + const turn = turnFor({ + input: [{ type: 'message', role: 'user', content: 'history' }], + instructions: 'You are a helpful assistant. Always mention the word quokka.', }); - if (!seenPayload) throw new Error('expected the upstream call to see the rewritten payload'); - // Original instructions preserved verbatim; SUMMARIZATION_PROMPT lives - // separately in the role=system input item at index 0. - assertEquals(seenPayload.instructions, 'You are a helpful assistant. Always mention the word quokka.'); - const rewrittenInput = seenPayload.input as ResponsesInputItem[]; - const compactorSystem = rewrittenInput[0] as { type: string; role: string; content: Array<{ text: string }> }; + assertEquals(turn.instructions, 'You are a helpful assistant. Always mention the word quokka.'); + const compactorSystem = itemsOf(turn)[0]!; assertEquals(compactorSystem.role, 'system'); - assertEquals(compactorSystem.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); + assertEquals(compactorSystem.content![0]!.text.includes('CONTEXT CHECKPOINT COMPACTION'), true); }); -test('compact + flag on: caller with no `instructions` still gets SUMMARIZATION_PROMPT as a role=system input item and instructions stays absent', async () => { +test('a caller with no `instructions` still gets the compactor prompt, and the slot stays absent', () => { // Baseline: when the caller sends no `instructions` field, the shim still // injects SUMMARIZATION_PROMPT via the role=system input item — matching // native compact, which always has its system-role compactor prompt in // scope regardless of whether the caller supplies a developer message. // The `instructions` slot stays unset so a downstream translator does not // synthesize a phantom developer message from thin air. - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); + const turn = turnFor({ input: [{ type: 'message', role: 'user', content: 'history' }] }); - let seenPayload: ResponsesPayload | undefined; - await withResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('irrelevant')(); - }); - - if (!seenPayload) throw new Error('expected the upstream call to see the rewritten payload'); - assertEquals(seenPayload.instructions, undefined); - const rewrittenInput = seenPayload.input as ResponsesInputItem[]; - const compactorSystem = rewrittenInput[0] as { type: string; role: string; content: Array<{ text: string }> }; + assertEquals(turn.instructions, undefined); + const compactorSystem = itemsOf(turn)[0]!; assertEquals(compactorSystem.type, 'message'); assertEquals(compactorSystem.role, 'system'); - assertEquals(compactorSystem.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); -}); - -test('compact + flag on: synthesized encrypted_content decodes to a user message containing the summary prefixed with the handoff prompt', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - const result = await withResponsesCompactShim(inv, stubCtx, fakeUpstreamRun('THE SUMMARY')); - if (result.type !== 'events') throw new Error('expected events branch'); - const collected = await collectResponsesProtocolEventsToResult(result.events); - const compactionItem = collected.output[0] as { type: string; encrypted_content: string }; - - // The encrypted_content decodes to our base64url-JSON marker: one - // user-role message carrying `${SUMMARY_PREFIX}\n${summary}` as - // input_text. The prefix rides inside the blob so a downstream LLM - // that echoes the compaction back reads the message as "another LLM's - // handoff summary", not as raw user speech. - const decoded = JSON.parse( - new TextDecoder().decode( - Uint8Array.from( - atob(compactionItem.encrypted_content.replace(/-/g, '+').replace(/_/g, '/')), - c => c.charCodeAt(0), - ), - ), - ); - assertEquals(decoded.length, 1); - assertEquals(decoded[0].type, 'message'); - assertEquals(decoded[0].role, 'user'); - assertEquals(decoded[0].content[0].type, 'input_text'); - assertEquals(decoded[0].content[0].text, `${SUMMARY_PREFIX}\nTHE SUMMARY`); -}); - -test('compact + flag on: upstream `output_text` SDK alias is dropped from the synthesized envelope', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - - // Some upstreams (and some OpenAPI implementations) emit the convenience - // `output_text` alias alongside `output`. The synthesized - // `response.compaction` envelope must not forward it — its value is the - // upstream's summary plaintext, which a downstream SDK reading - // `output_text` on a compaction envelope would surface in place of the - // opaque-blob contract `encrypted_content` is supposed to carry. - const runWithOutputText = (): Promise>> => { - const response: ResponsesResult = { - id: 'resp_fake_upstream', - object: 'response', - model: 'test-upstream-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'THE SUMMARY', annotations: [] }], - }], - output_text: 'THE SUMMARY', - error: null, - incomplete_details: null, - usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, - }; - return Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.completed', sequence_number: 0, response }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - )); - }; - const result = await withResponsesCompactShim(inv, stubCtx, runWithOutputText); - if (result.type !== 'events') throw new Error('expected events branch'); - const collected = await collectResponsesProtocolEventsToResult(result.events); - assertEquals(collected.output_text, undefined); + assertEquals(compactorSystem.content![0]!.text.includes('CONTEXT CHECKPOINT COMPACTION'), true); }); -test('compact + flag on: upstream incomplete status propagates onto the synthesized envelope', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - - // Simulate a summarization turn that hit `max_output_tokens` mid-stream: - // the upstream returns `status: 'incomplete'` with `incomplete_details` - // populated. The synthesized envelope must surface that — not pretend the - // turn ran to completion. - const runIncomplete = (): Promise>> => { - const response: ResponsesResult = { - id: 'resp_fake_upstream', - object: 'response', - model: 'test-upstream-model', - status: 'incomplete', - incomplete_details: { reason: 'max_output_tokens' }, - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'incomplete', - content: [{ type: 'output_text', text: 'partial summary', annotations: [] }], - }], - error: null, - usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, - }; - return Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.completed', sequence_number: 0, response }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - )); - }; - const result = await withResponsesCompactShim(inv, stubCtx, runIncomplete); - if (result.type !== 'events') throw new Error('expected events branch'); - const collected = await collectResponsesProtocolEventsToResult(result.events); - assertEquals(collected.status, 'incomplete'); - assertEquals(collected.incomplete_details, { reason: 'max_output_tokens' }); -}); - -test('compact + flag on: compaction_trigger items are stripped before the upstream call', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'real history' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { action: 'compact' }, - ); - - let seenPayload: ResponsesPayload | undefined; - await withResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('s')(); +test('compaction_trigger items are stripped from the history the compactor is sent', () => { + const turn = turnFor({ + input: [ + { type: 'message', role: 'user', content: 'real history' }, + { type: 'compaction_trigger' } as unknown as never, + ], }); - if (!seenPayload) throw new Error('expected the upstream call to fire'); - const items = seenPayload.input as Array<{ type: string }>; - assertEquals(items.every(i => i.type !== 'compaction_trigger'), true); + + assertEquals(itemsOf(turn).every(item => item.type !== 'compaction_trigger'), true); }); -test('compact + flag on: history ending on an assistant message gets a synthetic terminal user prompt appended', async () => { +test('a history ending on an assistant message gets a synthetic terminal user prompt appended', () => { // Anthropic Messages rejects assistant prefill: a conversation that // ends on an assistant turn returns 400 `This model does not support // assistant message prefill`. The shim normalizes by appending a // synthetic user-role nudge so the summarization call always ends on // a user message — harmless on OpenAI-style upstreams and load-bearing // for translated Anthropic ones. - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }, - { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hello back', annotations: [] }] } as unknown as never, - ], - }, - { action: 'compact' }, - ); - - let seenPayload: ResponsesPayload | undefined; - await withResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('s')(); + const turn = turnFor({ + input: [ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }, + { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hello back', annotations: [] }] } as unknown as never, + ], }); - if (!seenPayload) throw new Error('expected the upstream call to fire'); - const items = seenPayload.input as Array<{ type: string; role?: string; content?: Array<{ type: string; text: string }> }>; + // [SUMMARIZATION_PROMPT system, user 'hi', assistant 'hello back', synthetic user nudge] + const items = itemsOf(turn); assertEquals(items.length, 4); - assertEquals(items[0].role, 'system'); - const tail = items[items.length - 1]; + assertEquals(items[0]!.role, 'system'); + const tail = items[items.length - 1]!; assertEquals(tail.type, 'message'); assertEquals(tail.role, 'user'); - assertEquals(tail.content?.[0].type, 'input_text'); + assertEquals(tail.content![0]!.type, 'input_text'); }); -test('compact + flag off: passes through to run() unchanged', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'compact me' }] }, - { action: 'compact', flagOn: false }, - ); +// ── The summary, and the envelope it is packed into ────────────────────────── - let runCalled = false; - await withResponsesCompactShim(inv, stubCtx, () => { - runCalled = true; - return fakeUpstreamRun('unused')(); - }); - // Flag off: shim early-returns without expansion or pivot. The inner - // run() is called directly, action stays 'compact', payload unchanged. - assertEquals(runCalled, true); - assertEquals(inv.action, 'compact'); -}); - -// ── Bug 1 — engagement gating ──────────────────────────────────────────────── -// -// The shim engages when EITHER the per-upstream flag is on OR the candidate's -// targetApi is not 'responses'. The compact-shape check inside (action === -// 'compact' OR input contains compaction_trigger) decides whether to simulate -// or just pass the request through. Together these gates make the shim -// structurally required on non-Responses upstreams (Messages / Chat -// Completions translation has no `compaction_trigger` variant) while keeping -// the flag as the operator opt-in for Responses-target upstreams. - -test('generate + compaction_trigger + flag off + messages target: shim simulates (structurally required)', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'real history' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { action: 'generate', flagOn: false, targetApi: 'messages' }, - ); - - let seenPayload: ResponsesPayload | undefined; - const result = await withResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('CONDENSED SUMMARY')(); - }); +test('the summary is the item the turn closed, not the output its terminal stated', () => { + const message: ResponsesOutputItem = { + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'CONDENSED SUMMARY', annotations: [] }], + }; - // The shim should have engaged: the upstream sees the summarization - // prompt (as a role=system input item at the head of the history) and a - // stripped (no compaction_trigger) history, and the result is the - // synthesized `response.compaction` envelope. - if (!seenPayload) throw new Error('expected the upstream call to fire'); - const innerItems = seenPayload.input as ResponsesInputItem[]; - const head = innerItems[0] as { type: string; role: string; content: Array<{ text: string }> }; - assertEquals(head.type, 'message'); - assertEquals(head.role, 'system'); - assertEquals(head.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); - assertEquals(innerItems.every(i => i.type !== 'compaction_trigger'), true); - - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - const collected = await collectResponsesProtocolEventsToResult(result.events); - assertEquals(collected.object, 'response.compaction'); + assertEquals(summaryTextFrom(new Map([[0, message]]), []), 'CONDENSED SUMMARY'); + // A turn that closed nothing falls back to the terminal, as the client-facing egress does. + assertEquals(summaryTextFrom(new Map(), [message]), 'CONDENSED SUMMARY'); }); -test('generate + compaction_trigger + flag off + responses target: shim passes through (flag opt-in not taken)', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'real history' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { action: 'generate', flagOn: false, targetApi: 'responses' }, - ); - - let runCalled = false; - let seenAction: 'generate' | 'compact' | undefined; - let seenPayload: ResponsesPayload | undefined; - await withResponsesCompactShim(inv, stubCtx, () => { - runCalled = true; - seenAction = inv.action; - seenPayload = inv.payload; - return fakeUpstreamRun('unused')(); - }); +test('the synthesized encrypted_content decodes to a user message carrying the summary behind the handoff prefix', () => { + const envelope = buildCompactionEnvelope('cmp_1', 'THE SUMMARY', summarized('THE SUMMARY')); - // Shim did not engage — request flowed through untouched. - assertEquals(runCalled, true); - assertEquals(seenAction, 'generate'); - // Payload still carries the trigger (no expansion, no strip, no pivot). - const items = seenPayload?.input as Array<{ type: string }>; - assertEquals(items.some(i => i.type === 'compaction_trigger'), true); + assertEquals(envelope.object, 'response.compaction'); + const item = envelope.output[0] as unknown as { type: string; id: string; encrypted_content: string }; + assertEquals(item.type, 'compaction'); + assertEquals(item.id, 'cmp_1'); + // The prefix rides inside the blob so a downstream LLM that echoes the compaction back + // reads the message as "another LLM's handoff summary", not as raw user speech. + const decoded = decodeBlob(item.encrypted_content); + assertEquals(decoded.length, 1); + assertEquals(decoded[0]!.type, 'message'); + assertEquals(decoded[0]!.role, 'user'); + assertEquals(decoded[0]!.content[0]!.type, 'input_text'); + assertEquals(decoded[0]!.content[0]!.text, `${SUMMARY_PREFIX}\nTHE SUMMARY`); + // The envelope is answered under an id of this gateway's own rather than the summarization + // turn's. + assertEquals(envelope.id === 'resp_fake_upstream', false); }); -test('generate + compaction_trigger + flag on + messages target: shim simulates (same as flag-off path)', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'real history' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { action: 'generate', targetApi: 'messages' }, +test("the upstream's `output_text` SDK alias is dropped from the synthesized envelope", () => { + // Some upstreams (and some OpenAPI implementations) emit the convenience + // `output_text` alias alongside `output`. The synthesized + // `response.compaction` envelope must not forward it — its value is the + // upstream's summary plaintext, which a downstream SDK reading + // `output_text` on a compaction envelope would surface in place of the + // opaque-blob contract `encrypted_content` is supposed to carry. + const envelope = buildCompactionEnvelope( + 'cmp_1', + 'THE SUMMARY', + summarized('THE SUMMARY', { output_text: 'THE SUMMARY' } as Partial), ); - let seenPayload: ResponsesPayload | undefined; - const result = await withResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('CONDENSED SUMMARY')(); - }); - - if (!seenPayload) throw new Error('expected the upstream call to fire'); - const innerItems = seenPayload.input as ResponsesInputItem[]; - const head = innerItems[0] as { type: string; role: string; content: Array<{ text: string }> }; - assertEquals(head.role, 'system'); - assertEquals(head.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - const collected = await collectResponsesProtocolEventsToResult(result.events); - assertEquals(collected.object, 'response.compaction'); + assertEquals(envelope.output_text, undefined); }); -test('generate + flag on: runs inbound expansion but does not pivot', async () => { - const userItem = { type: 'message' as const, role: 'user' as const, content: 'expanded' }; - const encoded = encodeBase64UrlJson([userItem]); - const inv = makeInvocation( - { - input: [ - { type: 'compaction', id: 'cmp_1', encrypted_content: encoded } as unknown as never, - { type: 'message', role: 'user', content: 'follow-up' }, - ], - }, - { action: 'generate' }, - ); - - let runCalled = false; - await withResponsesCompactShim(inv, stubCtx, () => { - runCalled = true; - return fakeUpstreamRun('unused')(); - }); - assertEquals(runCalled, true); - // generate action stays as-is. - assertEquals(inv.action, 'generate'); - // Inbound expansion ran: the compaction item was replaced by `userItem`. - const items = inv.payload.input as Array<{ type: string; content?: unknown }>; - assertEquals(items.length, 2); - assertEquals(items[0], userItem); -}); - -test('compact + flag on: upstream api-error propagates', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - - const errorResult: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 502, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode('{"error":"upstream blew up"}'), - }; - const result = await withResponsesCompactShim(inv, stubCtx, () => Promise.resolve(errorResult)); - if (result.type !== 'api-error') throw new Error(`expected api-error, got ${result.type}`); - assertEquals(result.status, 502); +test('an incomplete summarization turn says so on the envelope it backs', () => { + // A summarization that hit `max_output_tokens` mid-stream comes back with + // `status: 'incomplete'` and `incomplete_details` populated. The synthesized + // envelope must surface that — not pretend the turn ran to completion. + const envelope = buildCompactionEnvelope('cmp_1', 'partial summary', summarized('partial summary', { + status: 'incomplete', + incomplete_details: { reason: 'max_output_tokens' }, + })); + + assertEquals(envelope.status, 'incomplete'); + assertEquals(envelope.incomplete_details, { reason: 'max_output_tokens' }); }); // ── Round-trip ──────────────────────────────────────────────────────────────── -test('round-trip: outbound synthesis then inbound expansion recovers the summary message prefixed with the handoff prompt', async () => { - // Step 1: simulate compaction — returns the synthesized envelope with - // shim-encoded `encrypted_content`. - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'long convo' }] }, - { action: 'compact' }, - ); - const result = await withResponsesCompactShim(inv, stubCtx, fakeUpstreamRun('SUMMARY TEXT')); - if (result.type !== 'events') throw new Error('expected events'); - const collected = await collectResponsesProtocolEventsToResult(result.events); - const compactionItem = collected.output[0] as { type: string; id?: string; encrypted_content: string }; - - // Step 2: next turn echoes the compaction item back as an input item; - // inbound expansion replaces it with the summary message — carrying the - // handoff prefix that was baked into the blob at encode time. - const nextTurn: CanonicalResponsesPayload = { - model: 'test-model', - input: [ - { type: 'compaction', id: compactionItem.id ?? 'cmp_rt', encrypted_content: compactionItem.encrypted_content } as unknown as never, - ], - }; - const expanded = expandShimCompactionItems(nextTurn); - const items = expanded.input as Array<{ type: string; role: string; content: Array<{ type: string; text: string }> }>; - assertEquals(items.length, 1); - assertEquals(items[0].type, 'message'); - assertEquals(items[0].role, 'user'); - assertEquals(items[0].content[0].text, `${SUMMARY_PREFIX}\nSUMMARY TEXT`); -}); +test('round-trip: what one turn packed is what the next turn is sent', () => { + const envelope = buildCompactionEnvelope('cmp_rt', 'SUMMARY TEXT', summarized('SUMMARY TEXT')); + const item = envelope.output[0] as unknown as { id: string; encrypted_content: string }; -const upstreamRunStatingNoOutput = (summaryText: string): () => Promise>> => { - const message = { - type: 'message' as const, - id: 'msg_1', - role: 'assistant' as const, - status: 'completed' as const, - content: [{ type: 'output_text' as const, text: summaryText, annotations: [] }], - }; - const response: ResponsesResult = { - id: 'resp_fake_upstream', - object: 'response', - model: 'test-upstream-model', - status: 'completed', - output: [], - error: null, - incomplete_details: null, - usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, - }; - return () => Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.output_item.added', sequence_number: 0, output_index: 0, item: message }); - yield eventFrame({ type: 'response.output_item.done', sequence_number: 1, output_index: 0, item: message }); - yield eventFrame({ type: 'response.completed', sequence_number: 2, response }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - )); -}; - -test('compact + flag on: the summary is the item the turn closed, not the output its terminal stated', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'long conversation history' }] }, - { action: 'compact' }, - ); - - const result = await withResponsesCompactShim(inv, stubCtx, upstreamRunStatingNoOutput('CONDENSED SUMMARY')); - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); + // The next turn echoes the compaction item back as an input item; expansion replaces it + // with the summary message, carrying the handoff prefix baked in at encode time. + const expanded = expandShimCompactionItems({ + model: 'test-model', + input: [{ type: 'compaction', id: item.id, encrypted_content: item.encrypted_content } as unknown as ResponsesInputItem], + }); - const collected = await collectResponsesProtocolEventsToResult(result.events); - const compaction = collected.output[0] as unknown as { type: string; encrypted_content: string }; - assertEquals(compaction.type, 'compaction'); - const decoded = JSON.parse(new TextDecoder().decode( - Uint8Array.from(atob(compaction.encrypted_content.replace(/-/g, '+').replace(/_/g, '/')), c => c.charCodeAt(0)), - )) as Array<{ content: Array<{ text: string }> }>; - assertEquals(decoded[0]?.content[0]?.text, `${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); + const items = expanded.input as { type: string; role: string; content: { type: string; text: string }[] }[]; + assertEquals(items.length, 1); + assertEquals(items[0]!.type, 'message'); + assertEquals(items[0]!.role, 'user'); + assertEquals(items[0]!.content[0]!.text, `${SUMMARY_PREFIX}\nSUMMARY TEXT`); }); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts index 4464cc2367..26e2463ba7 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts @@ -1,65 +1,40 @@ -// Compact-shim — simulates a `response.compaction` envelope against upstreams -// that have no native compaction wire. +// Compact-shim — what a `response.compaction` envelope is made of, where no upstream wire +// makes one. // -// Engagement is the OR of two conditions: -// 1. The per-upstream `responses-compact-shim` flag is on. This is the -// operator-controlled opt-in for Responses-target upstreams that -// already answer a compact request themselves — natively through -// `/responses/compact` (codex / azure / custom), or by replaying -// `RemoteCompactionV2` over `/responses` (copilot) — but where we still -// want shim-synthesized envelopes. -// 2. The candidate's `targetApi` is not `responses`. When the upstream is -// Messages or Chat Completions, the translation layer has no concept -// of a `compaction` output item or a `compaction_trigger` input item. -// The shim is structurally required regardless of the flag — without -// it, a `compaction_trigger` item would reach the translator and -// crash on the unknown variant. +// The rules that use these are stages: `simulatesCompaction` says which candidates' +// compactions are the shim's to make, `expandShimCompactions` puts a blob of ours back into +// the items it stood for, and `summarizeForCompaction` sends the turn built here and packs +// what comes back (`responses/pipeline.ts`). Both of this protocol's chains compose them, +// because a compaction the shim wrote is issued through one entry — `/v1/responses/compact`, +// or a generate turn whose input ends in a `compaction_trigger` — and echoed back into the +// other by the client that received it. // -// Inner compact-shape detection is also the OR of two conditions: -// - `invocation.action === 'compact'` (the native `/responses/compact` -// entry point), or -// - `invocation.payload.input` contains a `compaction_trigger` item -// (Codex CLI's RemoteCompactionV2 path: a `generate` call whose input -// ends in a control item that semantically requests compaction). +// What is here is the shim's own substance: // -// Flow when engaged and compact-shaped: -// 1. Inbound: walk `payload.input` for `compaction` items whose -// `encrypted_content` decodes as our base64url-JSON marker. Each match -// is replaced inline with the items it originally encoded — so a -// subsequent turn that echoes back the synthesized compaction sees the -// summarized history. -// 2. Outbound: pivot the action to 'generate', prepend a role=system -// message carrying the SUMMARIZATION_PROMPT (vendored from -// openai/codex), strip any `compaction_trigger` items, append a -// terminal user message if the history ends on a non-user item -// (Anthropic Messages rejects assistant prefill), and force -// `store: false` so the ephemeral summarization turn does not -// pollute the upstream's conversation history. The caller's -// `instructions` field flows through untouched — native -// `/responses/compact` keeps SUMMARIZATION_PROMPT as a system-role -// prompt AND forwards the caller's instructions as a developer-role -// message alongside, and we mirror that shape. Call `run()` to -// drive the chain through the normal generate path; collect the -// resulting summary text; pack a single user-role message -// containing the summary into a synthetic `response.compaction` -// envelope. Mutations of `ctx.payload` / `ctx.action` are one-way -// per the project's interceptor convention; attempt.invoke does -// not consume the post-chain `ctx` for its result shape (it keys -// envelope-drain off the caller's intent action). +// - `expandShimCompactionItems`: walk `payload.input` for `compaction` items whose +// `encrypted_content` decodes as our base64url-JSON marker, and replace each inline with +// the items it originally encoded — so a turn that echoes back a synthesized compaction +// is sent the summarized history rather than a blob. +// - `summarizationTurnFor`: the turn a compaction is simulated with. A role=system message +// carrying the SUMMARIZATION_PROMPT (vendored from openai/codex) at the head of the +// history, any `compaction_trigger` items stripped, a terminal user message appended +// where the history ends on a non-user item (Anthropic Messages rejects assistant +// prefill), and `store: false` so the ephemeral summarization turn does not pollute the +// upstream's conversation history. The caller's `instructions` field flows through +// untouched — native `/responses/compact` keeps SUMMARIZATION_PROMPT as a system-role +// prompt AND forwards the caller's instructions as a developer-role message alongside, +// and we mirror that shape. +// - `summaryTextFrom` and `buildCompactionEnvelope`: the summary the turn produced, and the +// synthetic `response.compaction` envelope carrying it as a blob only this gateway reads. // -// Foreign-upstream blobs (opaque strings that fail base64url+JSON decoding -// or fail the array-of-objects-with-string-types schema below) round-trip -// untouched, so the operator can selectively turn the flag off for the -// codex / copilot / azure / custom upstreams that answer compact themselves. +// Foreign-upstream blobs (opaque strings that fail base64url+JSON decoding or fail the +// array-of-objects-with-string-types schema below) round-trip untouched, so the operator can +// selectively turn the flag off for the codex / copilot / azure / custom upstreams that +// answer compact themselves. -import type { ResponsesInterceptor, ResponsesInvocation } from './types.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../shared/base64url-json.ts'; import { isJsonObject } from '../../../../shared/json-helpers.ts'; -import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; -import { syntheticEventsFromResult } from '../items/output.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectResponsesProtocolEventsToResult, createRandomResponsesItemId, type CanonicalResponsesPayload, type ResponsesInputItem, type ResponsesOutputItem, type ResponsesResult, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import { providerModelOf, type ExecuteResult } from '@floway-dev/provider'; +import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesOutputItem, ResponsesResult } from '@floway-dev/protocols/responses'; // The two vendored constants below (SUMMARIZATION_PROMPT and SUMMARY_PREFIX) // are the compactor system prompt and the handoff prefix openai/codex ships @@ -182,8 +157,6 @@ export const expandShimCompactionItems = (payload: CanonicalResponsesPayload): C // ── Outbound summarization ──────────────────────────────────────────────────── -type ChainRun = () => Promise>>; - // The spec makes the item lifecycle the authority and requires nothing of the // terminal's `output`; a Codex upstream states an `output` that omits the // assistant message it just closed. A turn that closed nothing falls back to @@ -320,68 +293,5 @@ export const summarizationTurnFor = (payload: CanonicalResponsesPayload): Canoni // silently discards the conversation. export const EMPTY_SUMMARY_MESSAGE = 'Responses compact shim: the summarization turn closed no assistant text to summarize'; -const simulateCompaction = async (ctx: ResponsesInvocation, gatewayCtx: ChatGatewayCtx, run: ChainRun): Promise>> => { - ctx.payload = summarizationTurnFor(ctx.payload); - // Pivot the action so the inner dispatch routes to the upstream's - // generate wire instead of its compact wire. The mutation is one-way: - // the project's interceptor convention is that every `ctx.*` write - // propagates downstream and is never restored on the way out. Post-chain - // consumers keep their own captured copies of inputs they care about. - ctx.action = 'generate'; - - const upstreamResult = await run(); - - if (upstreamResult.type !== 'events') { - // api-error / internal-error from the upstream propagate so the client - // learns the compaction failed rather than receiving a silent empty - // envelope. - return upstreamResult; - } - - const closedItems = new Map(); - const observed = (async function* (): AsyncIterable> { - for await (const frame of upstreamResult.events) { - if (frame.type === 'event' && frame.event.type === 'response.output_item.done') { - closedItems.set(frame.event.output_index, frame.event.item); - } - yield frame; - } - })(); - - const collected = await collectResponsesProtocolEventsToResult(observed); - const summaryText = summaryTextFrom(closedItems, collected.output); - if (summaryText.length === 0) throw new Error(EMPTY_SUMMARY_MESSAGE); - const cmpId = createRandomResponsesItemId('compaction'); - const synthesized = buildCompactionEnvelope(cmpId, summaryText, collected); - - // A real generate turn backs this envelope, so its own `incomplete` or - // `failed` status must reach the terminal event. - return { - ...upstreamResult, - events: syntheticEventsFromResult(synthesized), - }; -}; - export const containsCompactionTrigger = (input: readonly ResponsesInputItem[]): boolean => input.some(item => item.type === 'compaction_trigger'); - -export const withResponsesCompactShim: ResponsesInterceptor = async (ctx, gatewayCtx, run) => { - // The shim is engaged when the operator turned it on for this upstream, - // OR when the upstream's targetApi is not Responses (Messages / - // Chat Completions have no compaction wire and would crash on the - // unknown `compaction_trigger` input variant). - const flagOn = providerModelOf(ctx.candidate).enabledFlags.has('responses-compact-shim'); - const structurallyRequired = ctx.targetApi !== 'responses'; - if (!flagOn && !structurallyRequired) return await run(); - - // Inbound: expand any prior shim-encoded compactions back into their - // original items so the upstream sees the summarized history. - ctx.payload = expandShimCompactionItems(ctx.payload); - - // Compact-shaped requests are either the native `/responses/compact` - // action or a `generate` call whose input ends in a `compaction_trigger`. - const isCompactShaped = ctx.action === 'compact' || containsCompactionTrigger(ctx.payload.input); - if (!isCompactShaped) return await run(); - - return await simulateCompaction(ctx, gatewayCtx, run); -}; From c8445b517f401b655acfbda72578ff069e629a07 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:11:40 +0800 Subject: [PATCH 058/156] test(chat): drive /v1/chat/completions through the real route Fifty-nine files cover the chat families and two of them reach a route. That gap hid a total outage: every Copilot turn answered 502 because the record's frozen payload met a provider that writes into the body it is handed, and only a route test saw it. This states, against a Copilot upstream so the provider's own boundary chain runs, what the direct-call tests cannot: a streamed turn arrives as well-formed Chat Completions SSE ending in the sentinel, a collected turn arrives as one assembled body, an upstream's refusal arrives in the upstream's own words and status, a served turn writes exactly one usage row carrying the tokens the upstream reported, and a candidate reachable only over /messages still answers the client in the protocol it asked in. Each row was checked by breaking what it names: dropping the done frame from the SSE render, inverting the edge's stream/collect branch, synthesizing the refusal envelope over the upstream's body, billing empty quantities, and handing the target wire's frames up without the pair's mapping. --- .../chat/routes-chat-completions_test.ts | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts new file mode 100644 index 0000000000..ef61f494e5 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts @@ -0,0 +1,204 @@ +// `/v1/chat/completions`, from the client's socket to Copilot's and back. +// +// Everything else in this directory calls a chain, a stage or a helper directly. What no such +// test can state is that the route *works*: the parts it skips — the provider's own boundary +// interceptors, the fetcher, the epilogue that writes the client's bytes and settles what the +// stream billed — are exactly where a turn has been lost before. So each row here drives the +// real Hono route against a Copilot upstream, reads the bytes the client would read, and says +// something about them that a broken chain could not produce. + +import { expect, test } from 'vitest'; + +import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; +import { copilotModels, parseSSEText, requestApp, setupAppTest, sseChatCompletionsResponse, sseMessagesResponse } from '../../test-utils/app.ts'; +import { flushBackground } from '../../test-utils/background-tracker.ts'; +import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const ANSWER = 'hello from copilot'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer the row decides. */ +const withCopilot = async ( + models: Parameters[0], + serve: (path: string, body: Record) => Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return serve(url.pathname, await request.json() as Record); + }, + run, + ); + +/** The answer a Chat Completions upstream gives, as the SSE every chat endpoint really speaks. */ +const upstreamTurn = (model: string): Response => + sseChatCompletionsResponse({ + id: 'chatcmpl_route', + object: 'chat.completion', + created: 1_772_000_000, + model, + choices: [{ index: 0, message: { role: 'assistant', content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 11, completion_tokens: 7, total_tokens: 18 }, + }); + +const post = async (apiKey: string, body: Record): Promise => + await requestApp('/v1/chat/completions', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify(body), + }); + +interface ChatChunk { + readonly object: string; + readonly choices: readonly { readonly delta?: { readonly content?: string }; readonly finish_reason: string | null }[]; +} + +test('a streaming turn reaches the client as Chat Completions SSE, terminator included', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + (path, body) => { + expect(path).toBe('/chat/completions'); + // The provider forces streaming on every chat endpoint, so what the client asked for is + // not what the wire carries — and the marker below is the Copilot boundary chain having + // run on a body it was free to write into. + expect(body.stream).toBe(true); + return upstreamTurn('gpt-route'); + }, + async () => await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }], stream: true }), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('text/event-stream'); + + const frames = parseSSEText(await response.text()); + expect(frames.length > 1).toBe(true); + // The sentinel is what an OpenAI client reads as the end of the turn; a stream that stopped + // without it is a truncated answer presented as a whole one. + expect(frames.at(-1)?.data).toBe('[DONE]'); + + const chunks = frames.slice(0, -1).map(frame => JSON.parse(frame.data) as ChatChunk); + for (const chunk of chunks) expect(chunk.object).toBe('chat.completion.chunk'); + expect(chunks.map(chunk => chunk.choices[0]?.delta?.content ?? '').join('')).toBe(ANSWER); + expect(chunks.some(chunk => chunk.choices[0]?.finish_reason === 'stop')).toBe(true); +}); + +test('a non-streaming turn reaches the client as one assembled Chat Completions body', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => upstreamTurn('gpt-route'), + async () => await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }] }), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + + const body = await response.json() as { + object: string; + choices: { index: number; message: { role: string; content: string }; finish_reason: string }[]; + usage: { prompt_tokens: number; completion_tokens: number }; + }; + expect(body.object).toBe('chat.completion'); + expect(body.choices).toHaveLength(1); + expect(body.choices[0].message.role).toBe('assistant'); + expect(body.choices[0].message.content).toBe(ANSWER); + expect(body.choices[0].finish_reason).toBe('stop'); + // What the upstream metered is stated to the client too, and it is the same reading the + // usage row below is written from. + expect(body.usage.prompt_tokens).toBe(11); + expect(body.usage.completion_tokens).toBe(7); +}); + +test('an upstream refusal reaches the client in the upstream-s own words and status', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota', code: 'quota_exceeded' } }; + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => jsonResponse(refusal, 402), + async () => await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }] }), + ); + + expect(response.status).toBe(402); + // Not `{ error: { message, type: 'api_error' } }` — the answer to "why was I refused" is the + // refusing party's to give, so the gateway forwards it rather than quoting it. + expect(await response.json()).toEqual(refusal); +}); + +test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { + const { apiKey, repo } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => upstreamTurn('gpt-route'), + async () => { + const answer = await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }], stream: true }); + // A stream's numbers arrive with its last chunk, so the row is written after the client + // has read it — a test that never read the body would settle nothing. + await answer.text(); + return answer; + }, + ); + + expect(response.status).toBe(200); + await flushBackground(); + + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(rows[0].keyId).toBe(apiKey.id); + expect(rows[0].requests).toBe(1); + // The two regressions this fences: a row written with no metrics at all, and a row written + // with every quantity at zero. + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 11, output: 7 }); +}); + +test('a turn dialled over the Messages wire still answers the client in Chat Completions', async () => { + const { apiKey, repo } = await setupAppTest(); + + // The candidate serves `/messages` and nothing else, so the picker's first preference — this + // protocol's own wire — is unavailable and the turn leaves through the handoff. A chain that + // dialled anything else would call a Copilot method that has no upstream mocked here. + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + (path, body) => { + expect(path).toBe('/v1/messages'); + expect(body.stream).toBe(true); + return sseMessagesResponse({ + id: 'msg_route', + type: 'message', + role: 'assistant', + model: 'anthropic-route', + content: [{ type: 'text', text: ANSWER }], + stop_reason: 'end_turn', + usage: { input_tokens: 13, output_tokens: 5 }, + }); + }, + async () => await post(apiKey.key, { model: 'anthropic-route', messages: [{ role: 'user', content: 'hi' }] }), + ); + + expect(response.status).toBe(200); + const body = await response.json() as { + object: string; + choices: { message: { role: string; content: string }; finish_reason: string }[]; + }; + expect(body.object).toBe('chat.completion'); + expect(body.choices[0].message.role).toBe('assistant'); + expect(body.choices[0].message.content).toBe(ANSWER); + expect(body.choices[0].finish_reason).toBe('stop'); + + // The reading is taken on the dialect the upstream actually spoke, so a translated turn is + // metered like any other rather than billing nothing. + await flushBackground(); + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 13, output: 5 }); +}); From 38135fcd11dd83f307836bb79e6da79c5faac5fe Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:14:56 +0800 Subject: [PATCH 059/156] test(messages): drive /v1/messages through the real route The same statement the Chat Completions route file makes, for the protocol Claude Code speaks, and against a Copilot upstream so the provider's Messages boundary chain runs on a body it is free to write into. Two things are this family's own and are said rather than assumed: the turn ends with `message_stop` and not a transport sentinel, and every frame the client reads carries the SSE event name of the event its data states. The answer's text is found by block kind, because the edge writes the turn's affinity state back as a leading redacted_thinking block. Each row was checked by breaking what it names: dropping message_stop from the client's render, inverting the edge's stream/collect branch, synthesizing the refusal envelope over the upstream's body, billing empty quantities, and handing the target wire's frames up without the pair's mapping. --- .../data-plane/chat/routes-messages_test.ts | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts new file mode 100644 index 0000000000..de1820af7b --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts @@ -0,0 +1,213 @@ +// `/v1/messages`, from the client's socket to Copilot's and back. +// +// The same statement the Chat Completions route file makes, for the protocol Claude Code +// speaks. Two things are this family's own and are said here rather than assumed: Anthropic +// ends a turn with an event rather than a transport sentinel, so `message_stop` is what the +// terminal assertion looks for; and the edge writes the turn's own affinity state into the +// answer as a leading `redacted_thinking` block, so the text is found by kind rather than at +// index zero. + +import { expect, test } from 'vitest'; + +import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; +import { copilotModels, parseSSEText, requestApp, setupAppTest, sseChatCompletionsResponse, sseMessagesResponse } from '../../test-utils/app.ts'; +import { flushBackground } from '../../test-utils/background-tracker.ts'; +import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const ANSWER = 'hello from copilot'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer the row decides. */ +const withCopilot = async ( + models: Parameters[0], + serve: (path: string, body: Record) => Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return serve(url.pathname, await request.json() as Record); + }, + run, + ); + +/** The answer a Messages upstream gives, as the SSE every chat endpoint really speaks. */ +const upstreamTurn = (model: string): Response => + sseMessagesResponse({ + id: 'msg_route', + type: 'message', + role: 'assistant', + model, + content: [{ type: 'text', text: ANSWER }], + stop_reason: 'end_turn', + usage: { input_tokens: 17, output_tokens: 4 }, + }); + +const post = async (apiKey: string, body: Record): Promise => + await requestApp('/v1/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify(body), + }); + +const turn = (model: string, extra: Record = {}): Record => + ({ model, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }], ...extra }); + +interface MessagesBody { + readonly type: string; + readonly role: string; + readonly stop_reason: string; + readonly content: readonly { readonly type: string; readonly text?: string }[]; + readonly usage: { readonly input_tokens: number; readonly output_tokens: number }; +} + +/** The answer's own text, past the carrier the edge writes back. */ +const textOf = (body: MessagesBody): string | undefined => + body.content.find(block => block.type === 'text')?.text; + +test('a streaming turn reaches the client as Messages SSE, message_stop included', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + (path, body) => { + expect(path).toBe('/v1/messages'); + // The provider forces streaming on every chat endpoint, so what the client asked for is + // not what the wire carries. + expect(body.stream).toBe(true); + return upstreamTurn('anthropic-route'); + }, + async () => await post(apiKey.key, turn('anthropic-route', { stream: true })), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('text/event-stream'); + + const frames = parseSSEText(await response.text()); + // Anthropic names its own SSE events, and every frame the client reads carries the name of + // the event its data states. + for (const frame of frames) { + expect(frame.event).toBe((JSON.parse(frame.data) as { type: string }).type); + } + expect(frames[0].event).toBe('message_start'); + // The turn ends with an event rather than a transport sentinel — there is no `[DONE]` here, + // and a stream that stopped before this one is an answer nobody can act on. + expect(frames.at(-1)?.event).toBe('message_stop'); + + const text = frames + .filter(frame => frame.event === 'content_block_delta') + .map(frame => (JSON.parse(frame.data) as { delta: { text?: string } }).delta.text ?? '') + .join(''); + expect(text).toBe(ANSWER); +}); + +test('a non-streaming turn reaches the client as one assembled Messages body', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + () => upstreamTurn('anthropic-route'), + async () => await post(apiKey.key, turn('anthropic-route')), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + + const body = await response.json() as MessagesBody; + expect(body.type).toBe('message'); + expect(body.role).toBe('assistant'); + expect(body.stop_reason).toBe('end_turn'); + expect(textOf(body)).toBe(ANSWER); + // What the upstream metered is stated to the client too, and it is the same reading the + // usage row below is written from. + expect(body.usage.input_tokens).toBe(17); + expect(body.usage.output_tokens).toBe(4); +}); + +test('an upstream refusal reaches the client in the upstream-s own words and status', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { type: 'error', error: { type: 'permission_error', message: 'copilot said no' } }; + + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + () => jsonResponse(refusal, 403), + async () => await post(apiKey.key, turn('anthropic-route')), + ); + + expect(response.status).toBe(403); + // Not the gateway's own `{ type: 'error', error: { type: 'api_error', … } }` — the answer to + // "why was I refused" is the refusing party's to give. + expect(await response.json()).toEqual(refusal); +}); + +test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { + const { apiKey, repo } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + () => upstreamTurn('anthropic-route'), + async () => { + const answer = await post(apiKey.key, turn('anthropic-route', { stream: true })); + // A stream's numbers arrive with its last chunk, so the row is written after the client + // has read it — a test that never read the body would settle nothing. + await answer.text(); + return answer; + }, + ); + + expect(response.status).toBe(200); + await flushBackground(); + + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(rows[0].keyId).toBe(apiKey.id); + expect(rows[0].requests).toBe(1); + // The two regressions this fences: a row written with no metrics at all, and a row written + // with every quantity at zero. + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 17, output: 4 }); +}); + +test('a turn dialled over the Chat Completions wire still answers the client in Messages', async () => { + const { apiKey, repo } = await setupAppTest(); + + // The candidate serves `/chat/completions` and nothing else, so neither this protocol's own + // wire nor the Responses one is available and the turn leaves through the handoff. A chain + // that dialled anything else would call a Copilot method with no upstream mocked here. + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + (path, body) => { + expect(path).toBe('/chat/completions'); + // The usage chunk is asked for by the wire the turn landed on, not by the chain it + // arrived from — which is what keeps a translated turn metered at all. + expect(body.stream_options).toEqual({ include_usage: true }); + return sseChatCompletionsResponse({ + id: 'chatcmpl_route', + object: 'chat.completion', + created: 1_772_000_000, + model: 'gpt-route', + choices: [{ index: 0, message: { role: 'assistant', content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 9, completion_tokens: 6, total_tokens: 15 }, + }); + }, + async () => await post(apiKey.key, turn('gpt-route')), + ); + + expect(response.status).toBe(200); + const body = await response.json() as MessagesBody; + expect(body.type).toBe('message'); + expect(body.role).toBe('assistant'); + expect(body.stop_reason).toBe('end_turn'); + expect(textOf(body)).toBe(ANSWER); + + // The reading is taken on the dialect the upstream actually spoke, so a translated turn is + // metered like any other rather than billing nothing. + await flushBackground(); + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 9, output: 6 }); +}); From 4ae8c47d273516cba49262dfa7b72f21cf37b5af Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:15:37 +0800 Subject: [PATCH 060/156] fix(chat): refuse an untranslatable body in the client's own protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A translated wire runs the client's body through a translator, and a translator refuses input it would otherwise coerce or silently drop. Nothing caught that refusal, so it escaped the run: a Gemini turn carrying a `functionResponse` part in model content answered 500 with an internal-error envelope and a stack trace, naming the gateway as the party at fault for a request that was simply not serveable over that wire. The replaced surface rendered it as a 400 in the caller's own protocol, which is what `translatorInputErrorResult` was for. The handoff is the one place every translation happens, so it answers there. Writing only a status and a sentence is enough — each family already renders a failure in its own protocol at its own edge, which is the only place that knows what that protocol's clients read. Answering rather than throwing also makes the verdict a candidate's rather than the run's: the same body may translate cleanly for the next candidate, whose wire is another protocol, and failover re-runs the suffix to find out. That is something the replaced surface had no way to express. --- .../chat/translation-refusal_test.ts | 60 +++++++++++++++++++ .../gateway/src/data-plane/chat/handoff.ts | 46 +++++++++++++- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts new file mode 100644 index 0000000000..28919d8c60 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts @@ -0,0 +1,60 @@ +// A body the target protocol cannot represent is the client's fault. +// +// A translated wire runs the client's body through a translator, and a translator refuses input +// it would otherwise have to coerce or silently drop. The replaced surface rendered that refusal +// as a 400 in the client's own protocol; a run that let it escape as a throw would answer 500 with +// a stack trace, telling the caller that the gateway broke on a request that was simply not +// serveable over that wire. +// +// It is answered rather than thrown for a second reason the replaced surface had no way to +// express: the verdict belongs to one candidate. The next candidate may be reachable on a wire +// whose protocol *can* carry the body, and failover re-runs the suffix to find out. + +import { test } from 'vitest'; + +import { copilotModels, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const upstream = (models: Parameters[0]) => async (request: Request): Promise => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + throw new Error(`Unhandled fetch ${request.url}`); +}; + +// A `functionResponse` part in model content: a shape Gemini defines and Chat Completions has +// nowhere to put, so the translator refuses it rather than dropping it. +const untranslatableGeminiBody = { + contents: [ + { role: 'model', parts: [{ functionResponse: { name: 'f', response: { ok: true } } }] }, + { role: 'user', parts: [{ text: 'hi' }] }, + ], +}; + +test('a body the target protocol cannot carry is refused in the client own protocol', async () => { + const { apiKey } = await setupAppTest(); + + await withMockedFetch( + upstream([{ id: 'gpt-translate', supported_endpoints: ['/chat/completions'] }]), + async () => { + const response = await requestApp('/v1beta/models/gpt-translate:generateContent', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify(untranslatableGeminiBody), + }); + + // Google-RPC, not the gateway's internal-error envelope: `error.status` is the field a + // Gemini client reads, and a 500 would name the gateway as the party at fault. + assertEquals(response.status, 400); + const body = await response.json() as { error: { code: number; status: string; message: string } }; + assertEquals(body.error.code, 400); + assertEquals(body.error.status, 'INVALID_ARGUMENT'); + assertEquals(body.error.message.includes('not supported in model content'), true); + // The stack trace an internal failure carries has no business in a caller-input refusal. + assertEquals('stack' in body.error, false); + }, + ); +}); diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index 8e420677f2..e11547c6be 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -26,6 +26,7 @@ import { defineStage, move, type Pipeline, type Use } from '@floway-dev/pipeline import type { ModelEndpoints, ProtocolFrame } from '@floway-dev/protocols/common'; import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; import type { TranslateTripResult } from '@floway-dev/translate'; +import { TranslatorInputError } from '@floway-dev/translate'; type RequestKey = Extract; type ResponseKey = Extract; @@ -47,12 +48,34 @@ export interface Handoff Promise>; } +/** + * What a source chain holds when the translation itself refused. + * + * The four keys are what "no upstream was called" looks like on this family's slice: nothing + * billed, no upstream headers to carry, and nothing still streaming for settlement to wait on. + * The streamed-usage key is the response key's own sub-key by the fact space's naming rule, so + * naming it here costs no call site a parameter — and a family that broke the convention would + * be caught at assembly, where its real key would go unprovided and join the entry contract. + * + * No envelope is written. Each family renders a failure in its own protocol at its own edge, + * which is the only place that knows what that protocol's clients read. + */ +const refuseTranslation = (response: ResponseKey, message: string): Record => ({ + 'response.usage.billable': [], + 'response.http.headers': [], + [`${response}.streamedUsage`]: null, + [response]: { status: 400, message }, +}); + /** * Translates on the way down and back on the way up. * * An upstream refusal gets the pair's own rewrite where it has one: the canonical case is a * context-window error becoming the Anthropic `prompt is too long:` shape that Claude Code * reads for auto-compaction. A pair that declares no rewrite hands the refusal up unchanged. + * + * It carries the `return` trait too, because a body the target protocol cannot represent is + * answered here rather than dialled. */ export const handOff = ( handoff: Handoff, @@ -63,6 +86,14 @@ export const handOff = >({ name: `handOff:${handoff.from.request}→${handoff.to.request}`, + return: { + provides: [ + handoff.from.response, + `${handoff.from.response}.streamedUsage`, + 'response.usage.billable', + 'response.http.headers', + ], + }, through: { request: { needs: [handoff.from.request], @@ -80,7 +111,20 @@ export const handOff = { const record = facts as Record; const { [handoff.from.request]: source, ...down } = record; - const trip = await handoff.trip(source as ChatFacts[Source]); + + let trip; + try { + trip = await handoff.trip(source as ChatFacts[Source]); + } catch (error) { + // Input this target protocol cannot represent is the client's fault, and the translator + // is what knows which part of the body was at fault. Answering rather than throwing is + // also what keeps it a *candidate's* verdict: the same body may translate cleanly for + // the next candidate, whose wire is a different protocol, and failover re-runs the + // suffix to find out. Anything else raised down here is a fault of this gateway's and + // rides up as one. + if (!(error instanceof TranslatorInputError)) throw error; + return move({ ...down, ...refuseTranslation(handoff.from.response, error.message) }) as never; + } const back = await next({ ...down, [handoff.to.request]: move(trip.target) } as never) as Record; const { [handoff.to.response]: answered, ...up } = back; From 28cf1861ddfd93e59c338f0b0cdbd44ee6d490b8 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:19:46 +0800 Subject: [PATCH 061/156] test(gemini): drive :generateContent through the real route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same statement the other two route files make, for the one family with no wire of its own. Every row here is already a translated turn, so the last one adds the second pair — the same request answered over a Messages-only candidate — because "the client gets its own protocol back" has to hold for whichever wire the picker landed on. This family ends a turn differently from either OpenAI-shaped protocol: no sentinel, and the last frame is the one whose candidate states a finishReason. The refusal row also states what this family decided deliberately — the upstream's own object and status reach the client rather than being quoted back inside a Google-RPC envelope that cannot express either. Each row was checked by breaking what it names: dropping the terminal event from the client's render, inverting the edge's stream/collect branch, synthesizing the refusal envelope over the upstream's body, billing empty quantities on the wire the turn was dialled on, and narrowing the target picker so the Messages pair is unreachable. --- .../data-plane/chat/routes-gemini_test.ts | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts new file mode 100644 index 0000000000..fd02f7a6ed --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts @@ -0,0 +1,203 @@ +// `/v1beta/models/:model:generateContent`, from the client's socket to Copilot's and back. +// +// The same statement the other two route files make, for the one family with no wire of its +// own: the provider surface has no Gemini call, so every candidate is reached through a +// translation and each row here is already a translated turn. What the last row adds is the +// second pair — the same request answered over a Messages-only candidate — because "the client +// gets its own protocol back" has to hold for whichever wire the picker landed on. +// +// Gemini also ends a turn differently from either OpenAI-shaped protocol: there is no +// sentinel, and the last frame is the one whose candidate states a `finishReason`. + +import { expect, test } from 'vitest'; + +import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; +import { copilotModels, parseSSEText, requestApp, setupAppTest, sseChatCompletionsResponse, sseMessagesResponse } from '../../test-utils/app.ts'; +import { flushBackground } from '../../test-utils/background-tracker.ts'; +import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const ANSWER = 'hello from copilot'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer the row decides. */ +const withCopilot = async ( + models: Parameters[0], + serve: (path: string, body: Record) => Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return serve(url.pathname, await request.json() as Record); + }, + run, + ); + +/** The answer a Chat Completions upstream gives, as the SSE every chat endpoint really speaks. + * Gemini's first-preference wire, so this is what most of these turns are dialled on. */ +const chatCompletionsTurn = (): Response => + sseChatCompletionsResponse({ + id: 'chatcmpl_route', + object: 'chat.completion', + created: 1_772_000_000, + model: 'gpt-route', + choices: [{ index: 0, message: { role: 'assistant', content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 21, completion_tokens: 3, total_tokens: 24 }, + }); + +const post = async (apiKey: string, modelAction: string): Promise => + await requestApp(`/v1beta/models/${modelAction}`, { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }), + }); + +interface GeminiEvent { + readonly candidates?: readonly { + readonly content: { readonly parts: readonly { readonly text?: string }[] }; + readonly finishReason?: string; + }[]; +} + +const geminiText = (event: GeminiEvent): string => + (event.candidates ?? []).flatMap(candidate => candidate.content.parts.map(part => part.text ?? '')).join(''); + +test('a streaming turn reaches the client as Gemini SSE, ending on the frame that states a finish reason', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + (path, body) => { + expect(path).toBe('/chat/completions'); + // Gemini has no wire of its own, so what left the gateway is the translated turn — a + // Chat Completions body, streaming because every chat endpoint here does. + expect(body.messages).toBeDefined(); + expect(body.stream).toBe(true); + return chatCompletionsTurn(); + }, + async () => await post(apiKey.key, 'gpt-route:streamGenerateContent'), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('text/event-stream'); + + const frames = parseSSEText(await response.text()); + expect(frames.length > 0).toBe(true); + // One JSON object per `data:` line, and no sentinel — Gemini has none to write, so a `[DONE]` + // here would be a frame no Google client can parse. + const events = frames.map(frame => JSON.parse(frame.data) as GeminiEvent); + for (const frame of frames) expect(frame.event).toBe('message'); + expect(events.map(geminiText).join('')).toBe(ANSWER); + // The turn's end is stated on the frames the client is handed, not on the wire below them. + expect(events.at(-1)?.candidates?.[0]?.finishReason).toBe('STOP'); + expect(events.slice(0, -1).every(event => event.candidates?.[0]?.finishReason === undefined)).toBe(true); +}); + +test('a non-streaming turn reaches the client as one assembled Gemini body', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => chatCompletionsTurn(), + async () => await post(apiKey.key, 'gpt-route:generateContent'), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + + const body = await response.json() as GeminiEvent & { usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number } }; + expect(body.candidates).toHaveLength(1); + expect(body.candidates?.[0].finishReason).toBe('STOP'); + expect(geminiText(body)).toBe(ANSWER); + // What the upstream metered is stated to the client too, in this protocol's own names. + expect(body.usageMetadata?.promptTokenCount).toBe(21); + expect(body.usageMetadata?.candidatesTokenCount).toBe(3); +}); + +test('an upstream refusal reaches the client in the upstream-s own words and status', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota', code: 'quota_exceeded' } }; + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => jsonResponse(refusal, 402), + async () => await post(apiKey.key, 'gpt-route:generateContent'), + ); + + // The status the upstream sent, with none of the coercion a Google-RPC envelope would force + // onto the codes it cannot express. + expect(response.status).toBe(402); + // And the upstream's own object, rather than the same words quoted back inside + // `{ error: { code, message, status } }`. + expect(await response.json()).toEqual(refusal); +}); + +test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { + const { apiKey, repo } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => chatCompletionsTurn(), + async () => { + const answer = await post(apiKey.key, 'gpt-route:streamGenerateContent'); + // A stream's numbers arrive with its last chunk, so the row is written after the client + // has read it — a test that never read the body would settle nothing. + await answer.text(); + return answer; + }, + ); + + expect(response.status).toBe(200); + await flushBackground(); + + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(rows[0].keyId).toBe(apiKey.id); + expect(rows[0].requests).toBe(1); + // The reading is taken on the dialect the upstream actually spoke, which for this family is + // always a translated one — and the two regressions this fences are a row written with no + // metrics at all and a row written with every quantity at zero. + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 21, output: 3 }); +}); + +test('a turn dialled over the Messages wire still answers the client in Gemini', async () => { + const { apiKey, repo } = await setupAppTest(); + + // The candidate serves `/messages` and nothing else, so the picker's first preference is + // unavailable and the turn leaves through the other pair. A chain that dialled anything else + // would call a Copilot method with no upstream mocked here. + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + (path, body) => { + expect(path).toBe('/v1/messages'); + expect(body.messages).toBeDefined(); + expect(body.stream).toBe(true); + return sseMessagesResponse({ + id: 'msg_route', + type: 'message', + role: 'assistant', + model: 'anthropic-route', + content: [{ type: 'text', text: ANSWER }], + stop_reason: 'end_turn', + usage: { input_tokens: 8, output_tokens: 2 }, + }); + }, + async () => await post(apiKey.key, 'anthropic-route:generateContent'), + ); + + expect(response.status).toBe(200); + const body = await response.json() as GeminiEvent; + expect(body.candidates).toHaveLength(1); + expect(body.candidates?.[0].finishReason).toBe('STOP'); + expect(geminiText(body)).toBe(ANSWER); + + await flushBackground(); + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 8, output: 2 }); +}); From ba6d14820c9f473c89f2d248b75b495bf3a9471b Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:19:54 +0800 Subject: [PATCH 062/156] refactor(chat): delete the refusal renderers the pipeline replaced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `renderMessagesFailure` and `renderGeminiFailure` rendered a pre-stream `ChatServeFailure` into the old execute surface's `ExecuteResult`. Narrowing now states its own refusal and the edge renders it, and the failed-upstream list those two appended is appended by `resolveChatCandidates` — so both, and the result builders under them, describe an architecture that is gone. `translatorInputErrorResult` went the same way, one step behind: the handoff now answers a translator's refusal itself. Its test kept two properties worth keeping — the Anthropic envelope's shape and its key order, which a client comparing bytes against Anthropic-direct depends on — so the test moves to the function that writes that envelope today rather than being deleted with the one that used to. --- .../data-plane/chat/messages/errors_test.ts | 51 ++++++++------- .../src/data-plane/chat/gemini/errors.ts | 46 ------------- .../src/data-plane/chat/messages/errors.ts | 64 ++----------------- 3 files changed, 33 insertions(+), 128 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/messages/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/errors_test.ts index 1ec9368328..84f53f8f44 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages/errors_test.ts @@ -1,37 +1,42 @@ +// The Anthropic error envelope, at the function that writes it. +// +// Key order is load-bearing: a client comparing this gateway's refusal against +// Anthropic-direct's compares bytes, so `type`, then `error`, then a top-level `request_id`. + import { test } from 'vitest'; -import { translatorInputErrorResult } from '../../../../src/data-plane/chat/messages/errors.ts'; -import type { ApiErrorResult } from '@floway-dev/provider'; +import { renderMessagesError } from '../../../../src/data-plane/chat/messages/errors.ts'; import { assert, assertEquals } from '@floway-dev/test-utils'; -import { TranslatorInputError } from '@floway-dev/translate'; - -const apiErrorOf = (result: ReturnType): ApiErrorResult => result as ApiErrorResult; -const bodyOf = (result: ReturnType): Record => - JSON.parse(new TextDecoder().decode(apiErrorOf(result).body)); -test('translatorInputErrorResult renders an Anthropic 400 invalid_request_error envelope with top-level request_id', () => { - const result = translatorInputErrorResult( - new TranslatorInputError("Invalid 'image_url' content part in system or developer message. Only 'text' content parts are supported in system messages on this model."), +test('renders an Anthropic invalid_request_error envelope with a top-level request_id', () => { + const body = renderMessagesError( + 400, + "Invalid 'image_url' content part in system or developer message. Only 'text' content parts are supported in system messages on this model.", ); - const apiError = apiErrorOf(result); - assertEquals(apiError.type, 'api-error'); - assertEquals(apiError.source, 'gateway'); - assertEquals(apiError.status, 400); - assertEquals(apiError.headers.get('content-type'), 'application/json'); - - const body = bodyOf(result); assertEquals(body.type, 'error'); assertEquals(body.error, { type: 'invalid_request_error', message: "Invalid 'image_url' content part in system or developer message. Only 'text' content parts are supported in system messages on this model.", }); - assert(typeof body.request_id === 'string' && /^req_[A-Za-z0-9]{24}$/.test(body.request_id), `request_id ${String(body.request_id)} must match Anthropic-shape req_<24 opaque chars>`); + assert( + typeof body.request_id === 'string' && /^req_[A-Za-z0-9]{24}$/.test(body.request_id), + `request_id ${String(body.request_id)} must match Anthropic-shape req_<24 opaque chars>`, + ); +}); + +test('preserves Anthropic key order: type, error, request_id', () => { + const raw = JSON.stringify(renderMessagesError(400, 'whatever')); + assert( + /^\{"type":"error","error":\{"type":"invalid_request_error","message":"whatever"\},"request_id":"req_[A-Za-z0-9]{24}"\}$/.test(raw), + `body ${raw} must match Anthropic-direct byte shape`, + ); }); -test('translatorInputErrorResult preserves Anthropic key order: type, error, request_id', () => { - const result = translatorInputErrorResult(new TranslatorInputError('whatever')); - const raw = new TextDecoder().decode(apiErrorOf(result).body); - // Key-order is load-bearing for byte-faithfulness with Anthropic-direct. - assert(/^\{"type":"error","error":\{"type":"invalid_request_error","message":"whatever"\},"request_id":"req_[A-Za-z0-9]{24}"\}$/.test(raw), `body ${raw} must match Anthropic-direct byte shape`); +test('names the condition each status carries in this protocol', () => { + // The status a refusal was made with is what selects the type name, so the mapping is what + // makes a gateway-made refusal read like the upstream's own. + assertEquals((renderMessagesError(429, 'slow down').error as { type: string }).type, 'rate_limit_error'); + assertEquals((renderMessagesError(529, 'overloaded').error as { type: string }).type, 'overloaded_error'); + assertEquals((renderMessagesError(502, 'bad gateway').error as { type: string }).type, 'api_error'); }); diff --git a/packages/gateway/src/data-plane/chat/gemini/errors.ts b/packages/gateway/src/data-plane/chat/gemini/errors.ts index b9def82698..c69c92c6a2 100644 --- a/packages/gateway/src/data-plane/chat/gemini/errors.ts +++ b/packages/gateway/src/data-plane/chat/gemini/errors.ts @@ -1,10 +1,3 @@ -import { appendFailedUpstreams } from '../../shared/failed-upstreams.ts'; -import type { ChatServeFailure } from '../shared/errors.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import type { ExecuteResult, PerformanceTelemetryContext } from '@floway-dev/provider'; -import type { TranslatorInputError } from '@floway-dev/translate'; - // Google RPC Status envelope, used by Gemini's `error` channel everywhere // (HTTP body, SSE-tunnelled error event). /** What a Gemini client is sent when a turn produced no content. `error.status` is the @@ -35,42 +28,3 @@ export const geminiStatusForHttpStatus = (status: number): string => { return 'INTERNAL'; } }; - -const geminiRpcErrorResult = (status: number, message: string, performance?: PerformanceTelemetryContext): ExecuteResult> => ({ - type: 'api-error', - source: 'gateway', - status, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(JSON.stringify({ - error: { code: status, message, status: geminiStatusForHttpStatus(status) }, - })), - ...(performance ? { performance } : {}), -}); - -// Translator surfaced a caller-input violation (unsupported content part, -// disallowed role, missing required field, etc.). Render as a 400 -// INVALID_ARGUMENT envelope so the caller sees a Gemini-shaped failure -// instead of the internal-error 500 envelope. `performance` carries the -// throwing candidate's telemetry attribution when the throw fired -// mid-attempt (see AttemptState.telemetry). -export const translatorInputErrorResult = ( - error: TranslatorInputError, - performance?: PerformanceTelemetryContext, -): ExecuteResult> => - geminiRpcErrorResult(400, error.message, performance); - -// `endpoint` selects between `:generateContent` and `:countTokens` only in -// the `model-unsupported` message string. -export const renderGeminiFailure = ( - failure: ChatServeFailure, - endpoint: 'generate' | 'countTokens', -): ExecuteResult> => { - switch (failure.kind) { - case 'routing-unavailable': - return geminiRpcErrorResult(400, failure.message); - case 'model-missing': - return geminiRpcErrorResult(404, appendFailedUpstreams(`Model ${failure.model} is not available on any configured upstream.`, failure.failedUpstreams)); - case 'model-unsupported': - return geminiRpcErrorResult(400, appendFailedUpstreams(`Model ${failure.model} does not support ${endpoint === 'countTokens' ? 'countTokens' : 'the Gemini generateContent endpoint'}.`, failure.failedUpstreams)); - } -}; diff --git a/packages/gateway/src/data-plane/chat/messages/errors.ts b/packages/gateway/src/data-plane/chat/messages/errors.ts index 9dccb76536..889d6b11b6 100644 --- a/packages/gateway/src/data-plane/chat/messages/errors.ts +++ b/packages/gateway/src/data-plane/chat/messages/errors.ts @@ -1,16 +1,9 @@ -import { appendFailedUpstreams } from '../../shared/failed-upstreams.ts'; -import type { ChatServeFailure } from '../shared/errors.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { generateAnthropicId, type MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ExecuteResult, PerformanceTelemetryContext } from '@floway-dev/provider'; -import type { TranslatorInputError } from '@floway-dev/translate'; +import { generateAnthropicId } from '@floway-dev/protocols/messages'; -// Anthropic Messages error envelope used to render pre-stream -// `ChatServeFailure`s. These are gateway-synthesized rather than received -// from any upstream — `source: 'gateway'` so the dump labels them as such. -// Byte-shape matches Anthropic-direct: `{type:'error', error:{type, message}, -// request_id}` with `request_id` at the top level (alongside `error`, not -// nested inside it) and key order load-bearing for byte-faithfulness. +// Anthropic's own error envelope. The byte shape matches Anthropic-direct: +// `{type:'error', error:{type, message}, request_id}` with `request_id` at the +// top level — alongside `error`, not nested inside it — and the key order is +// load-bearing for byte-faithfulness. /** What a Messages client is sent when a turn produced no content. The envelope is * Anthropic's own — `type` at the top, the error's own `type` inside, and a `request_id` a * caller quotes back in a support thread — so it is the protocol's shape rather than the @@ -34,50 +27,3 @@ const anthropicErrorTypeForStatus = (status: number): string => { default: return status >= 500 ? 'api_error' : 'invalid_request_error'; } }; - -const anthropicErrorResult = ( - status: number, - type: string, - message: string, - performance?: PerformanceTelemetryContext, -): ExecuteResult> => ({ - type: 'api-error', - source: 'gateway', - status, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(JSON.stringify({ - type: 'error', - error: { type, message }, - request_id: generateAnthropicId('req'), - })), - ...(performance ? { performance } : {}), -}); - -// Translator surfaced a caller-input violation (unsupported content part, -// disallowed role, missing required field, etc.). Render as a 400 -// invalid_request_error so the caller sees a protocol-shaped failure -// instead of the internal-error 502 envelope. `performance` carries the -// throwing candidate's telemetry attribution when the throw fired -// mid-attempt (see AttemptState.telemetry). -export const translatorInputErrorResult = ( - error: TranslatorInputError, - performance?: PerformanceTelemetryContext, -): ExecuteResult> => - anthropicErrorResult(400, 'invalid_request_error', error.message, performance); - -// `endpoint` selects between `/messages` and `/messages/count_tokens` only in -// the `model-unsupported` message string. -export const renderMessagesFailure = ( - failure: ChatServeFailure, - endpoint: 'generate' | 'countTokens', -): ExecuteResult> => { - const endpointPath = endpoint === 'countTokens' ? '/messages/count_tokens' : '/messages'; - switch (failure.kind) { - case 'routing-unavailable': - return anthropicErrorResult(400, 'invalid_request_error', failure.message); - case 'model-missing': - return anthropicErrorResult(404, 'not_found_error', appendFailedUpstreams(`Model ${failure.model} is not available on any configured upstream.`, failure.failedUpstreams)); - case 'model-unsupported': - return anthropicErrorResult(400, 'invalid_request_error', appendFailedUpstreams(`Model ${failure.model} does not support the ${endpointPath} endpoint.`, failure.failedUpstreams)); - } -}; From cf9ce7ddab2089e3dd9de35ea19ce8676d7c6942 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:20:40 +0800 Subject: [PATCH 063/156] fix(responses): hand a provider a body it owns at both Responses dials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two dials this family makes were the last building their own body: a spread of the record's request, the candidate's model stamped on it, the addressed id dropped. That gives a provider a fresh top level over frozen children, and a provider shapes the body it is handed in place — Copilot marks individual messages for caching — so the first nested write throws and the client is answered 502 from the point in the stack least able to explain it. Both go through `bodyForAttempt`, as generation and measurement already do on the other families. What each dial still says for itself is what is specific to it: the compaction endpoint takes neither `store` nor `stream`. Copilot's Responses boundary happens to be written functionally today, so the route this adds says the turn reaches Copilot's own endpoint and comes back — what it rewrote on the way out included. What pins the ownership is a stage-level turn at each dial, dialling a provider that writes into a nested node the way the boundaries that already do. --- .../data-plane/chat/responses-compact_test.ts | 18 +++ .../chat/responses-pipeline_test.ts | 19 ++++ .../data-plane/chat/responses-routes_test.ts | 107 ++++++++++++++++++ .../src/data-plane/chat/responses/compact.ts | 5 +- .../src/data-plane/chat/responses/pipeline.ts | 9 +- 5 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts index 5eae1dfa69..9d1429c0ed 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts @@ -389,4 +389,22 @@ describe('the responses compaction chain', () => { { quantities: { input_tokens: '12', output_tokens: '3' } }, ]); }); + + // A provider owns the body it is dialled with and shapes it in place, down to nested nodes — + // Copilot marks individual items for caching — while the record that body is built from is + // deep-frozen. A dial that handed over a shallow copy throws at the first nested write, and + // this operation would answer 502 where the upstream had said nothing at all. + it('hands the provider a body it can write into, children included', async () => { + resolves([candidate({ + callResponses: async (model, body, action) => { + const item = (body as { input: Record[] }).input[0]!; + item.copilot_cache_control = { type: 'ephemeral' }; + return await compacts({})(model, body, action); + }, + })]); + + const { facts } = await compact(); + + expect(facts['response.http.status']).toBe(200); + }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index 9a01ce58c8..90405edd73 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -539,6 +539,25 @@ describe('the responses chain', () => { expect(facts['response.http.status']).toBe(200); }); + // A provider owns the body it is dialled with and shapes it in place, down to nested nodes — + // Copilot marks individual items for caching, and its Messages boundary writes into a nested + // field — while the record that body is built from is deep-frozen. A dial that handed over a + // shallow copy satisfies every type in the system and throws at the first nested write, + // answering the client with a 502 raised where nothing can explain it. + it('hands the provider a body it can write into, children included', async () => { + resolves([candidate({ + callResponses: async (_model, body) => { + const item = (body as { input: Record[] }).input[0]!; + item.copilot_cache_control = { type: 'ephemeral' }; + return stream(completed('hi')); + }, + })]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(200); + }); + // Codex asks for a compaction inside an ordinary turn, by ending its input with the control // item that requests one. Where the shim is this candidate's to run, that item never reaches // the upstream: what is sent instead is the compactor's own turn, and what comes back is an diff --git a/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts new file mode 100644 index 0000000000..3c9b712edc --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts @@ -0,0 +1,107 @@ +// `/v1/responses`, from the client's socket to Copilot's and back. +// +// Everything else in this directory calls a chain, a stage or a helper directly, against a +// provider that is a stub. What no such test can state is that the *dial* works: a provider's +// own boundary shapes the body it is handed — Copilot rewrites tool definitions, strips the +// fields its endpoint rejects, compresses inline images and re-stamps item ids — and the +// record that body is built from is deep-frozen. A body handed over as a shallow copy +// satisfies every type in the system and throws at the first write into a nested node, +// answering the client with a 502 raised where nothing can explain it. So this drives the +// real route against a Copilot upstream and reads what the client would read. + +import { expect, test } from 'vitest'; + +import { copilotModels, requestApp, setupAppTest, sseResponsesResponse } from '../../test-utils/app.ts'; +import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const ANSWER = 'hello from copilot'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer the row decides. */ +const withCopilot = async ( + models: Parameters[0], + serve: (path: string, body: Record) => Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return serve(url.pathname, await request.json() as Record); + }, + run, + ); + +const post = async (apiKey: string, body: Record): Promise => + await requestApp('/v1/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify(body), + }); + +test('a turn reaches Copilot over its own Responses endpoint and comes back as one response', async () => { + const { apiKey } = await setupAppTest(); + + let sent: Record | undefined; + const response = await withCopilot( + [{ id: 'gpt-responses', supported_endpoints: ['/responses'] }], + (path, body) => { + expect(path).toBe('/responses'); + sent = body; + return sseResponsesResponse({ + id: 'resp_route', + object: 'response', + model: 'gpt-responses', + status: 'completed', + output: [{ + type: 'message', id: 'msg_route', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text: ANSWER, annotations: [] }], + }], + error: null, + incomplete_details: null, + usage: { input_tokens: 11, output_tokens: 7, total_tokens: 18 }, + }); + }, + async () => await post(apiKey.key, { + model: 'gpt-responses', + // Everything the Copilot boundary has an opinion about, so the body it was handed is one + // it rewrote rather than one it merely forwarded: a namespace tool whose empty + // description it fills, an image-generation tool its endpoint rejects, a service tier it + // does not expose, and the snapshot hint it refuses. + input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }], + tools: [ + { type: 'namespace', name: 'repo', description: '', tools: [] }, + { type: 'image_generation' }, + ], + service_tier: 'flex', + store: true, + }), + ); + + // The regression this fences is a 502 whose body reads `Cannot add property …, object is not + // extensible`: the provider writing into a body whose nested nodes were still the record's. + expect(response.status).toBe(200); + + const body = await response.json() as { + object: string; + status: string; + output: { type: string; content?: { type: string; text: string }[] }[]; + usage: { input_tokens: number; output_tokens: number }; + }; + expect(body.object).toBe('response'); + expect(body.status).toBe('completed'); + expect(body.output.find(item => item.type === 'message')?.content?.[0]?.text).toBe(ANSWER); + expect(body.usage).toMatchObject({ input_tokens: 11, output_tokens: 7 }); + + // What the boundary made of the turn, on the wire: the tool its endpoint rejects is gone, + // the description it needs is filled, and neither field it refuses travels. + const tools = sent?.tools as { type: string; description?: string }[]; + expect(tools.map(tool => tool.type)).toEqual(['namespace']); + expect(tools[0]!.description).toBe('Tools in the repo namespace.'); + expect(sent).not.toHaveProperty('service_tier'); + expect(sent).toMatchObject({ store: false }); +}); diff --git a/packages/gateway/src/data-plane/chat/responses/compact.ts b/packages/gateway/src/data-plane/chat/responses/compact.ts index 92bb3dc5dc..46e3ee89af 100644 --- a/packages/gateway/src/data-plane/chat/responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/responses/compact.ts @@ -59,6 +59,7 @@ import { type ResponsesFacts, } from './pipeline.ts'; import { billableUsageFromResponsesResult } from './usage.ts'; +import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; @@ -216,12 +217,10 @@ const callResponsesCompactUpstream = defineStage< use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; - const payload = { ...asked, model: candidate.model.id }; - if (candidate.rules !== undefined) applyRulesToUpstreamResponses(payload, candidate.rules); // Neither field belongs on this endpoint: `store` is a gateway-only snapshot hint the // compaction endpoint rejects, and `stream` describes how an answer would be delivered // where there is one body and no stream. - const { model: _addressed, stream: _stream, store: _store, ...body } = payload; + const { stream: _stream, store: _store, ...body } = bodyForAttempt(asked, candidate, applyRulesToUpstreamResponses); let result; try { diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index a10d819dfc..ba9e3173b3 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -57,6 +57,7 @@ import { normalizeAssistantInputText } from './items/normalize-assistant-content import { syntheticEventsFromResult } from './items/output.ts'; import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve-prep.ts'; import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; +import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { AttemptSelector, BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; @@ -353,16 +354,12 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< // every stage between the fork and here has rewritten it — or, on a translated wire, what // the handoff put here. Client-carried state — an encrypted reasoning blob, a compaction // the upstream issued — was rewritten for the upstream that will see it, which is the - // whole reason a turn can be pinned at all. The id the client addressed does not travel — - // the provider re-stamps whatever it resolved upstream — and an alias' own rules apply to - // the body that is sent. + // whole reason a turn can be pinned at all. // // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to, which is the one a wire takes. const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; - const payload = { ...asked, model: candidate.model.id }; - if (candidate.rules !== undefined) applyRulesToUpstreamResponses(payload, candidate.rules); - const { model: _addressed, ...body } = payload; + const body = bodyForAttempt(asked, candidate, applyRulesToUpstreamResponses); let result; try { From aac415cf0137193df654caa9282fe4ce9277674c Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:23:03 +0800 Subject: [PATCH 064/156] docs(copilot): stop naming an array the pipeline removed The note explaining why the web-search shim is not registered on this boundary pointed at the gateway's `messagesInterceptors`, which no longer exists. What it was reaching for survives the rewrite: the shim is the gateway's work rather than a provider boundary's, and this provider opts into it by flag. --- .../src/interceptors/messages/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/provider-copilot/src/interceptors/messages/index.ts b/packages/provider-copilot/src/interceptors/messages/index.ts index 82791df1de..713f10e1d0 100644 --- a/packages/provider-copilot/src/interceptors/messages/index.ts +++ b/packages/provider-copilot/src/interceptors/messages/index.ts @@ -45,10 +45,12 @@ import type { CopilotMessagesBoundaryInterceptor, CopilotMessagesCountTokensBoun // compact-tagged value above — that mirrors the pre-boundary target-side // override. // -// `withMessagesWebSearchShim` is intentionally NOT registered here. It runs -// in the gateway's `messagesInterceptors` (filtered by enabled flags); the -// Copilot provider opts in by listing `messages-web-search-shim` in its -// default flag set (see COPILOT_DEFAULT_FLAGS in ../../defaults.ts). +// `withMessagesWebSearchShim` is intentionally NOT registered here. The shim +// is the gateway's rather than this provider's — it rewrites Anthropic's +// native server tool into an ordinary client tool and runs the search itself, +// which is work no provider boundary does. This provider opts into it by +// listing `messages-web-search-shim` in its default flag set (see +// COPILOT_DEFAULT_FLAGS in ../../defaults.ts). export const COPILOT_MESSAGES_BOUNDARY = [ rewriteContextWindowError, withCompactHeadersSet, From 88d7cc8b51eb1e9cb6fa47aed523cd82c855ad67 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:26:48 +0800 Subject: [PATCH 065/156] test(chat): refuse a streaming turn with a body at all three routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refusal rows already here send a turn that asked for one object. The riskier half is the turn that asked for SSE: the seam has a stream to open and must not, because nothing was ever generated and an error written into a 200 stream reads to every one of these clients as a turn that succeeded and said nothing. Verified against the current upstreams — the client is answered with the upstream's own object under its own status on all three — and each row was checked by making its family's edge render a failure as frames whenever the client asked to stream. --- .../{interceptors => }/compact-shim_test.ts | 0 .../chat/routes-chat-completions_test.ts | 18 ++++++++++++++++++ .../data-plane/chat/routes-gemini_test.ts | 18 ++++++++++++++++++ .../data-plane/chat/routes-messages_test.ts | 18 ++++++++++++++++++ .../{interceptors => }/compact-shim.ts | 0 5 files changed, 54 insertions(+) rename packages/gateway/__tests__/data-plane/chat/responses/{interceptors => }/compact-shim_test.ts (100%) rename packages/gateway/src/data-plane/chat/responses/{interceptors => }/compact-shim.ts (100%) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/compact-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/compact-shim_test.ts similarity index 100% rename from packages/gateway/__tests__/data-plane/chat/responses/interceptors/compact-shim_test.ts rename to packages/gateway/__tests__/data-plane/chat/responses/compact-shim_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts index ef61f494e5..65f0825da9 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts @@ -134,6 +134,24 @@ test('an upstream refusal reaches the client in the upstream-s own words and sta expect(await response.json()).toEqual(refusal); }); +test('a refusal on a turn that asked to stream is a body, not an opened stream', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota' } }; + + // The riskier half of the same statement: the client asked for SSE, so the seam has a + // stream to open and must not — nothing was ever generated, and an error written into a + // 200 stream is one an OpenAI client reads as a successful empty turn. + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => jsonResponse(refusal, 402), + async () => await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }], stream: true }), + ); + + expect(response.status).toBe(402); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + expect(await response.json()).toEqual(refusal); +}); + test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { const { apiKey, repo } = await setupAppTest(); diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts index fd02f7a6ed..3f1dff2c51 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts @@ -137,6 +137,24 @@ test('an upstream refusal reaches the client in the upstream-s own words and sta expect(await response.json()).toEqual(refusal); }); +test('a refusal on a turn that asked to stream is a body, not an opened stream', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota' } }; + + // The riskier half of the same statement: the client asked for SSE, so the seam has a + // stream to open and must not — nothing was ever generated, and a Gemini stream that + // carried the refusal would be read as a turn that produced no candidates. + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => jsonResponse(refusal, 402), + async () => await post(apiKey.key, 'gpt-route:streamGenerateContent'), + ); + + expect(response.status).toBe(402); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + expect(await response.json()).toEqual(refusal); +}); + test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { const { apiKey, repo } = await setupAppTest(); diff --git a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts index de1820af7b..267d3cd206 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts @@ -145,6 +145,24 @@ test('an upstream refusal reaches the client in the upstream-s own words and sta expect(await response.json()).toEqual(refusal); }); +test('a refusal on a turn that asked to stream is a body, not an opened stream', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { type: 'error', error: { type: 'permission_error', message: 'copilot said no' } }; + + // The riskier half of the same statement: the client asked for SSE, so the seam has a + // stream to open and must not — nothing was ever generated, and Claude Code reads an error + // written into a 200 stream as a turn that succeeded and said nothing. + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + () => jsonResponse(refusal, 403), + async () => await post(apiKey.key, turn('anthropic-route', { stream: true })), + ); + + expect(response.status).toBe(403); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + expect(await response.json()).toEqual(refusal); +}); + test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { const { apiKey, repo } = await setupAppTest(); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts b/packages/gateway/src/data-plane/chat/responses/compact-shim.ts similarity index 100% rename from packages/gateway/src/data-plane/chat/responses/interceptors/compact-shim.ts rename to packages/gateway/src/data-plane/chat/responses/compact-shim.ts From 2c0e2bac21d26be888ae348e9089895a1e3e0e35 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:36:40 +0800 Subject: [PATCH 066/156] refactor(chat): let the chat files say what the pipeline left them as MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four things the migration made true and nothing had written down. The compact shim moved out of `interceptors/` — nothing in it is an interceptor any more, and the rules that use it are stages in both chains that compose it. The Qwen normalizer's interceptor form went entirely: the stage beside it is what the wire runs, and its flag gate is now asserted where the stage is tested rather than only where the interceptor was. `renderResponsesFailure` and `responsesInputErrorResult` followed the two that went with the other families, and `openAiErrorResult` under them. With them gone, `ChatServeFailure` had two variants nobody could produce — narrowing states a model that is missing or unsupported as its own refusal now, failed upstreams and all — so the type is the one shape that still travels as a throw, and the affinity alias that extracted it is the type itself. The two Responses edges render through `renderFailure` like every other. --- .../data-plane/chat-interceptors_test.ts | 12 ++++ .../data-plane/chat/responses-compact_test.ts | 2 +- .../chat/responses-pipeline_test.ts | 2 +- .../chat/responses/compact-shim_test.ts | 4 +- .../data-plane/chat/responses/errors_test.ts | 58 ++--------------- .../vendor-qwen-normalize_test.ts | 63 ------------------- .../data-plane/chat/shared/errors_test.ts | 26 +++----- .../data-plane/chat/responses/compact-shim.ts | 4 +- .../src/data-plane/chat/responses/compact.ts | 10 +-- .../src/data-plane/chat/responses/errors.ts | 34 ++-------- .../interceptors/vendor-qwen-normalize.ts | 29 --------- .../src/data-plane/chat/responses/pipeline.ts | 16 ++--- .../chat/shared/affinity/selection.ts | 4 +- .../src/data-plane/chat/shared/errors.ts | 53 +++------------- 14 files changed, 61 insertions(+), 256 deletions(-) delete mode 100644 packages/gateway/__tests__/data-plane/chat/responses/interceptors/vendor-qwen-normalize_test.ts delete mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/vendor-qwen-normalize.ts diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index 13ede213ae..f95745b5e0 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -995,6 +995,18 @@ describe('the Responses interceptors, as stages', () => { expect(after.enable_thinking).toBe(false); }); + it('leaves the sentinel alone for an upstream that is not Qwen', async () => { + // The dialect is the flag's, not the payload's: the same sentinel means something else on + // the next upstream, and a rewrite that fired without the flag would put a Qwen field on a + // wire that has never heard of it. + const original = responsesPayload([], { reasoning: { effort: 'none' } }); + const { down } = await runStage(vendorQwenNormalizeForResponses, 'response.chat.responses', { + 'request.chat.responses': original, + 'route.attempt': attemptWith(), + }); + expect(down['request.chat.responses']).toBe(original); + }); + it('leaves a payload that asked for real reasoning alone, by identity', async () => { const original = responsesPayload([], { reasoning: { effort: 'high' } }); for (const stage of [vendorDeepSeekNormalizeForResponses, vendorQwenNormalizeForResponses]) { diff --git a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts index 9d1429c0ed..1735aec739 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts @@ -9,8 +9,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/responses/compact-shim.ts'; import { responsesCompactPipeline } from '../../../src/data-plane/chat/responses/compact.ts'; -import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/responses/interceptors/compact-shim.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index 90405edd73..eb014d02db 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -11,7 +11,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/responses/interceptors/compact-shim.ts'; +import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/responses/compact-shim.ts'; import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; diff --git a/packages/gateway/__tests__/data-plane/chat/responses/compact-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/compact-shim_test.ts index 82e9e83d09..2048e79d03 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/compact-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/compact-shim_test.ts @@ -11,8 +11,8 @@ import { summarizationTurnFor, summaryTextFrom, SUMMARY_PREFIX, -} from '../../../../../src/data-plane/chat/responses/interceptors/compact-shim.ts'; -import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../../src/shared/base64url-json.ts'; +} from '../../../../src/data-plane/chat/responses/compact-shim.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../src/shared/base64url-json.ts'; import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesOutputItem, ResponsesResult } from '@floway-dev/protocols/responses'; import { assertEquals } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/responses/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/errors_test.ts index 5acd50f080..03c580fb1b 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/errors_test.ts @@ -1,63 +1,15 @@ +// The Responses-only refusal, through the same throw/catch. Its shape is the reason the +// mechanism is generic over the failure type: the store finds the missing item deep inside a +// walk, and the stage that started the walk is what answers. + import { test } from 'vitest'; -import { responsesInputErrorResult, type ResponsesServeFailure } from '../../../../src/data-plane/chat/responses/errors.ts'; +import type { ResponsesServeFailure } from '../../../../src/data-plane/chat/responses/errors.ts'; import { throwChatServeFailure, tryCatchChatServeFailure } from '../../../../src/data-plane/chat/shared/errors.ts'; -import type { ApiErrorResult } from '@floway-dev/provider'; import { assertEquals, assertThrows } from '@floway-dev/test-utils'; -import { TranslatorInputError } from '@floway-dev/translate'; - -const bodyOf = (result: ApiErrorResult): unknown => - JSON.parse(new TextDecoder().decode(result.body)); test('round-trips the Responses-only item-not-found failure through throw/catch', () => { const failure: ResponsesServeFailure = { kind: 'item-not-found', itemId: 'msg_abc' }; const error = assertThrows(() => throwChatServeFailure(failure)); assertEquals(tryCatchChatServeFailure(error), failure); }); - -test('responsesInputErrorResult renders an OpenAI 400 invalid_request_error envelope with default `input` param', () => { - const result = responsesInputErrorResult( - new TranslatorInputError("Invalid input item type 'image_generation_call'."), - ); - assertEquals(result.type, 'api-error'); - assertEquals(result.source, 'gateway'); - assertEquals(result.status, 400); - assertEquals(bodyOf(result), { - error: { - message: "Invalid input item type 'image_generation_call'.", - type: 'invalid_request_error', - param: 'input', - code: null, - }, - }); -}); - -test('responsesInputErrorResult carries an explicit error code into the envelope', () => { - const result = responsesInputErrorResult( - new TranslatorInputError("Missing required parameter: 'model'.", { param: 'model', code: 'missing_required_parameter' }), - ); - - assertEquals(bodyOf(result), { - error: { - message: "Missing required parameter: 'model'.", - type: 'invalid_request_error', - param: 'model', - code: 'missing_required_parameter', - }, - }); -}); - -test('responsesInputErrorResult honors an explicit input param', () => { - const result = responsesInputErrorResult( - new TranslatorInputError('content block not supported', { param: 'input[1].content[0]' }), - ); - - assertEquals(bodyOf(result), { - error: { - message: 'content block not supported', - type: 'invalid_request_error', - param: 'input[1].content[0]', - code: null, - }, - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/vendor-qwen-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/vendor-qwen-normalize_test.ts deleted file mode 100644 index 009d4f00c5..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/vendor-qwen-normalize_test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { test } from 'vitest'; - -import type { ResponsesInvocation } from '../../../../../src/data-plane/chat/responses/interceptors/types.ts'; -import { withVendorQwenResponsesNormalize } from '../../../../../src/data-plane/chat/responses/interceptors/vendor-qwen-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => - Promise.resolve( - eventResult( - (async function* () { - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ), - ); - -const invocation = (payload: CanonicalResponsesPayload, enabledFlags: ReadonlySet = new Set(['vendor-qwen'])): ResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'responses', - headers: new Headers(), - action: 'generate', -}); - -test("vendor-qwen translates canonical reasoning.effort: 'none' into top-level enable_thinking:false", async () => { - const input = invocation({ - model: 'qwen-max', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'none' }, - }); - - await withVendorQwenResponsesNormalize(input, stubCtx, okEvents); - - const out = input.payload as unknown as Record; - assertEquals(out.reasoning, undefined); - assertEquals(out.enable_thinking, false); -}); - -test('vendor-qwen leaves a real reasoning.effort value untouched (only the none sentinel triggers the rewrite)', async () => { - const input = invocation({ model: 'qwen-max', input: [{ type: 'message', role: 'user', content: 'hi' }], reasoning: { effort: 'high' } }); - - await withVendorQwenResponsesNormalize(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'high' }); - const out = input.payload as unknown as Record; - assertEquals(out.enable_thinking, undefined); -}); - -test('vendor-qwen early-returns when its flag is not set on the candidate', async () => { - const input = invocation({ model: 'qwen-max', input: [{ type: 'message', role: 'user', content: 'hi' }], reasoning: { effort: 'none' } }, new Set()); - - await withVendorQwenResponsesNormalize(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'none' }); - const out = input.payload as unknown as Record; - assertEquals(out.enable_thinking, undefined); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/errors_test.ts index 9e77a7c03a..42945dadb8 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/errors_test.ts @@ -1,25 +1,17 @@ +// The one refusal that travels as a throw, and the guard that keeps it from catching anything +// else. What it carries has to survive the trip intact: the message names which carried target +// went missing, and only the selection several frames down knows that. + import { test } from 'vitest'; import { type ChatServeFailure, throwChatServeFailure, tryCatchChatServeFailure } from '../../../../src/data-plane/chat/shared/errors.ts'; import { assertEquals, assertThrows } from '@floway-dev/test-utils'; -const cases: readonly ChatServeFailure[] = [ - { kind: 'model-missing', model: 'gpt-9', failedUpstreams: [] }, - { kind: 'model-missing', model: 'gpt-9', failedUpstreams: ['Azure prod'] }, - { kind: 'model-unsupported', model: 'gpt-9', failedUpstreams: [] }, - { kind: 'model-unsupported', model: 'gpt-9', failedUpstreams: ['Azure prod', 'Custom'] }, - { kind: 'routing-unavailable', message: 'no upstream can serve this' }, -]; - -for (const failure of cases) { - const label = 'failedUpstreams' in failure && failure.failedUpstreams.length - ? `${failure.kind} (with ${failure.failedUpstreams.length} failed upstream(s))` - : failure.kind; - test(`round-trips ${label} through throw/catch`, () => { - const error = assertThrows(() => throwChatServeFailure(failure)); - assertEquals(tryCatchChatServeFailure(error), failure); - }); -} +test('round-trips routing-unavailable through throw/catch', () => { + const failure: ChatServeFailure = { kind: 'routing-unavailable', message: 'no upstream can serve this' }; + const error = assertThrows(() => throwChatServeFailure(failure)); + assertEquals(tryCatchChatServeFailure(error), failure); +}); test('returns null for an error not raised by throwChatServeFailure', () => { assertEquals(tryCatchChatServeFailure(new Error('something else')), null); diff --git a/packages/gateway/src/data-plane/chat/responses/compact-shim.ts b/packages/gateway/src/data-plane/chat/responses/compact-shim.ts index 26e2463ba7..51d982b843 100644 --- a/packages/gateway/src/data-plane/chat/responses/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/compact-shim.ts @@ -32,8 +32,8 @@ // selectively turn the flag off for the codex / copilot / azure / custom upstreams that // answer compact themselves. -import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../shared/base64url-json.ts'; -import { isJsonObject } from '../../../../shared/json-helpers.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../shared/base64url-json.ts'; +import { isJsonObject } from '../../../shared/json-helpers.ts'; import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesOutputItem, ResponsesResult } from '@floway-dev/protocols/responses'; // The two vendored constants below (SUMMARIZATION_PROMPT and SUMMARY_PREFIX) diff --git a/packages/gateway/src/data-plane/chat/responses/compact.ts b/packages/gateway/src/data-plane/chat/responses/compact.ts index 46e3ee89af..65ac1ff10c 100644 --- a/packages/gateway/src/data-plane/chat/responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/responses/compact.ts @@ -61,7 +61,7 @@ import { import { billableUsageFromResponsesResult } from './usage.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; -import { isFailure } from '../../pipeline/facts.ts'; +import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -78,7 +78,7 @@ import { import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; import { materializeAttempt, resolveChatCandidates, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; -import { renderProtocolError, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, type CanonicalResponsesPayload, @@ -139,9 +139,9 @@ const emitResponsesCompaction = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(renderProtocolError( - answer.body, - () => answer.envelope ?? { error: { message: answer.message, type: 'api_error' } }, + 'response.chat.responses.rendered': move(renderFailure( + answer, + () => ({ error: { message: answer.message, type: 'api_error' } }), )), 'response.http.status': answer.status, }; diff --git a/packages/gateway/src/data-plane/chat/responses/errors.ts b/packages/gateway/src/data-plane/chat/responses/errors.ts index 38c5f7e341..374fbd0457 100644 --- a/packages/gateway/src/data-plane/chat/responses/errors.ts +++ b/packages/gateway/src/data-plane/chat/responses/errors.ts @@ -1,31 +1,7 @@ -import { appendFailedUpstreams } from '../../shared/failed-upstreams.ts'; -import { openAiErrorResult, type ChatServeFailure } from '../shared/errors.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import type { ApiErrorResult, ExecuteResult, PerformanceTelemetryContext } from '@floway-dev/provider'; +import type { ChatServeFailure } from '../shared/errors.ts'; +/** The chat-wide refusals plus the one only this protocol can make: a turn that named an item + * by id which the store cannot resolve. It is thrown from inside the membrane's hydration + * rather than returned, because hydration walks the input and the item that is missing is + * found several frames below where the answer is written. */ export type ResponsesServeFailure = ChatServeFailure | { readonly kind: 'item-not-found'; readonly itemId: string }; - -// Caller-input violations discovered by translation or the source affinity -// membrane share the Responses 400 envelope. `performance` retains candidate -// attribution when validation fires after attempt dispatch. -export const responsesInputErrorResult = ( - error: { readonly message: string; readonly param?: string; readonly code?: string }, - performance?: PerformanceTelemetryContext, -): ApiErrorResult => - openAiErrorResult(400, error.message, { param: error.param ?? 'input', code: error.code ?? null }, performance); - -export const renderResponsesFailure = ( - failure: ResponsesServeFailure, -): ExecuteResult> => { - switch (failure.kind) { - case 'item-not-found': - return openAiErrorResult(404, `Item with id '${failure.itemId}' not found.`, { param: 'input', code: null }); - case 'routing-unavailable': - return openAiErrorResult(400, failure.message, { param: 'input', code: 'responses_item_routing_unavailable' }); - case 'model-missing': - return openAiErrorResult(404, appendFailedUpstreams(`Model ${failure.model} is not available on any configured upstream.`, failure.failedUpstreams)); - case 'model-unsupported': - return openAiErrorResult(400, appendFailedUpstreams(`Model ${failure.model} does not support the /responses endpoint.`, failure.failedUpstreams)); - } -}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/vendor-qwen-normalize.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/vendor-qwen-normalize.ts deleted file mode 100644 index 4ceb2d61ee..0000000000 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/vendor-qwen-normalize.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Qwen wire-dialect normalizer for the Responses protocol. Always-attached; -// flag-gated by `vendor-qwen`. Runs last among interceptors so it has the -// final say on the outbound wire body. -// -// Outbound (request → upstream): -// -// - `reasoning.effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel. Qwen uses a top-level `enable_thinking: false` field instead. -// We strip the entire `reasoning` object and emit the Qwen form. -// -// Inbound: nothing today. -// -// Reference: -// - https://www.alibabacloud.com/help/en/model-studio/deep-thinking - -import type { ResponsesInterceptor } from './types.ts'; -import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withVendorQwenResponsesNormalize: ResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-qwen')) return await run(); - - if (ctx.payload.reasoning?.effort === 'none') { - const { reasoning: _stripped, ...rest } = ctx.payload; - ctx.payload = { ...rest, enable_thinking: false } as CanonicalResponsesPayload; - } - - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index ba9e3173b3..c180cb578d 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -43,7 +43,6 @@ import { analyzeResponsesAffinity } from './affinity/ingress.ts'; import { wrapResponsesClientEgress } from './client-output.ts'; -import type { ResponsesServeFailure } from './errors.ts'; import { buildCompactionEnvelope, containsCompactionTrigger, @@ -51,7 +50,8 @@ import { expandShimCompactionItems, summarizationTurnFor, summaryTextFrom, -} from './interceptors/compact-shim.ts'; +} from './compact-shim.ts'; +import type { ResponsesServeFailure } from './errors.ts'; import { hydrateResponsesPayload } from './items/hydrate.ts'; import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; import { syntheticEventsFromResult } from './items/output.ts'; @@ -59,7 +59,7 @@ import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { AttemptSelector, BillableEntity } from '../../pipeline/facts.ts'; -import { isFailure } from '../../pipeline/facts.ts'; +import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -86,7 +86,7 @@ import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage, type Use } from '@floway-dev/pipeline'; -import { doneFrame, eventFrame, renderProtocolError, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { doneFrame, eventFrame, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, createRandomResponsesItemId, @@ -180,9 +180,9 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(renderProtocolError( - answer.body, - () => answer.envelope ?? { error: { message: answer.message, type: 'api_error' } }, + 'response.chat.responses.rendered': move(renderFailure( + answer, + () => ({ error: { message: answer.message, type: 'api_error' } }), )), 'response.http.status': answer.status, }; @@ -780,7 +780,7 @@ export const beginStoredAttempt = defineStage< // a generate turn's input with a `compaction_trigger`, `/v1/responses/compact` asks for one // by being called at all, and either way the envelope comes back to the client, who echoes it // into the ordinary turns that follow. What the shim is, what it vendors and what it cannot -// reproduce is at `interceptors/compact-shim.ts`. +// reproduce is at `compact-shim.ts`. /** * Whether this candidate's compactions are the shim's to simulate. diff --git a/packages/gateway/src/data-plane/chat/shared/affinity/selection.ts b/packages/gateway/src/data-plane/chat/shared/affinity/selection.ts index 035ea94b0d..3eaa3ec94a 100644 --- a/packages/gateway/src/data-plane/chat/shared/affinity/selection.ts +++ b/packages/gateway/src/data-plane/chat/shared/affinity/selection.ts @@ -18,8 +18,6 @@ export interface AffinityCandidateSelection { readonly payloadFor: (candidate: ModelCandidate) => T; } -export type AffinitySelectionFailure = Extract; - export type OptionalAffinityBlobProjection = | { readonly kind: 'preserve'; readonly value: string } | { readonly kind: 'remove'; readonly degrades: boolean }; @@ -104,7 +102,7 @@ export const defineAffinityRequest = ( export const selectAffinityCandidates = ( candidates: readonly ModelCandidate[], affinity: AffinityRequestAnalysis, -): AffinityCandidateSelection | AffinitySelectionFailure => { +): AffinityCandidateSelection | ChatServeFailure => { if (affinity.requiredTargets.length > 1) { return { kind: 'routing-unavailable', diff --git a/packages/gateway/src/data-plane/chat/shared/errors.ts b/packages/gateway/src/data-plane/chat/shared/errors.ts index 65bb54c9f0..c73f78ac9e 100644 --- a/packages/gateway/src/data-plane/chat/shared/errors.ts +++ b/packages/gateway/src/data-plane/chat/shared/errors.ts @@ -1,16 +1,14 @@ -import type { ApiErrorResult, PerformanceTelemetryContext } from '@floway-dev/provider'; +// Failures a chat protocol makes before it reaches an upstream, as a throw the chain that can +// answer them catches. Unexpected throws bubble as-is. +// +// Almost every refusal is a value on the record — narrowing states its own, and the edge +// renders it. What is left here is the one shape that cannot be: a fault found several frames +// inside a walk, where the answer is written by the stage that started the walk. -// Failures a chat protocol can render before reaching an upstream; unexpected -// throws bubble as-is. `failedUpstreams` on model-{missing,unsupported} -// carries the upstream names whose catalog fetch threw during this -// resolution — surfaced parenthetically so the caller can tell a genuine -// "no upstream has this model" miss from a transient outage where the -// upstream that owns the model is currently unreachable. Empty means -// every consulted upstream returned a catalog. -export type ChatServeFailure = - | { readonly kind: 'model-missing'; readonly model: string; readonly failedUpstreams: readonly string[] } - | { readonly kind: 'model-unsupported'; readonly model: string; readonly failedUpstreams: readonly string[] } - | { readonly kind: 'routing-unavailable'; readonly message: string }; +/** `routing-unavailable` is what affinity says when a turn carries state for an upstream no + * candidate can be routed to. Its message is the caller's to read: only the selection knows + * which of the carried targets went missing. */ +export type ChatServeFailure = { readonly kind: 'routing-unavailable'; readonly message: string }; class ChatServeFailureError extends Error { readonly failure: TFailure; @@ -27,34 +25,3 @@ export const throwChatServeFailure = (error: unknown): TFailure | null => error instanceof ChatServeFailureError ? error.failure as TFailure : null; - -export const openAiErrorResult = ( - status: number, - message: string, - extra?: { readonly param: string; readonly code: string | null }, - performance?: PerformanceTelemetryContext, -): ApiErrorResult => ({ - type: 'api-error', - source: 'gateway', - status, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(JSON.stringify({ - error: { message, type: 'invalid_request_error', ...extra }, - })), - ...(performance ? { performance } : {}), -}) satisfies ApiErrorResult; - -// Builds the failure value every serve dispatches with after `canServe` has -// dropped every candidate: `sawModel=true` means the inbound id exists in -// some upstream's catalog but no upstream wire reaches it for this protocol, -// rendered as 400 model-unsupported; `sawModel=false` means no upstream knows -// the id at all, rendered as 404 model-missing. The per-protocol failure -// renderer turns the value into its own envelope shape. -export const noViableCandidateFailure = ( - sawModel: boolean, - model: string, - failedUpstreams: readonly string[], -): ChatServeFailure => - sawModel - ? { kind: 'model-unsupported', model, failedUpstreams } - : { kind: 'model-missing', model, failedUpstreams }; From f4e3c3770329da1833365d5ae5d6210c7f5b9120 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 12:38:56 +0800 Subject: [PATCH 067/156] docs(chat): give every vendor constant a source, and correct one claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five constants encoded a vendor fact with nothing to check it against. Each now carries the primary source it was verified from. One of them was wrong. The billing-attribution stage said the block carries a per-call `cch=` and that Anthropic's endpoint reads it to bill against the user's plan tier. Reading the vendor's own shipped build (2.1.226): `cch` is the fixed literal `00000` and is emitted only on the first-party and Vertex paths, so on the third-party path this stage serves there is no `cch=` at all — what varies per call is the fingerprint suffix on `cc_version`, derived from the first user message. The line pattern was always what carried the load. And no source anywhere says the endpoint bills from it; what the build shows is a client-attribution signal, so the comment now says that much and no more. The rest were confirmed. DeepSeek documents `prompt_tokens` as exactly hit + miss. The stripped Gemini part fields are unforwardable for a reason Google states outright — a `fileUri` names a file the API will not hand back. The stripped tool capabilities turn out to be precisely the `Tool` schema minus `functionDeclarations`, which is worth saying because it tells the next reader how to re-derive the list rather than guess at it. --- .../src/data-plane/chat/interceptors.ts | 41 ++++++++++++++++--- .../interceptors/answer-claude-code-probe.ts | 11 +++-- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index bec66cccc9..ed56c6a5b9 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -641,7 +641,10 @@ const deepSeekInboundDeltas = (chunk: ChatCompletionsStreamEvent): ChatCompletio }; /** DeepSeek's "hit" count is the cached prefix length, which is what OpenAI's - * `cached_tokens` means; the "miss" count is what is left of the input and is dropped. */ + * `cached_tokens` means; the "miss" count is what is left of the input and is dropped. + * `prompt_tokens` is documented as exactly hit + miss, and the cache matches only a prefix. + * https://api-docs.deepseek.com/api/create-chat-completion + * https://api-docs.deepseek.com/guides/kv_cache */ const DEEPSEEK_CACHE_FIELDS = ['prompt_cache_hit_tokens', 'prompt_cache_miss_tokens'] as const; const deepSeekInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { @@ -854,11 +857,20 @@ const withMessagesReasoningDisabled = (payload: MessagesPayload): MessagesPayloa /** * Scrubs Claude Code's billing-attribution block out of the system prompt. * - * The block carries a `cch=` that flips per call, so an upstream that reads it as - * ordinary prompt text sees a different prompt every turn and never reuses its prompt cache. - * The Claude Code subscription endpoint reads it to bill the request against the user's plan - * tier and wants it intact, which is what the flag is for: it is on for the upstreams that - * treat the block as text and off for the one that treats it as billing. + * The block leads every Claude Code turn as `system[0]`: + * `x-anthropic-billing-header: cc_version=.; cc_entrypoint=;` + * with optional `cch=`, `cc_workload=` and `cc_prev_req=` segments after it. Two of those parts + * vary per call — the version's trailing fingerprint is derived from the first user message, + * and `cch=` was a per-request nonce on builds through 2.1.177 — so an upstream reading the + * block as ordinary prompt text sees a different prompt every turn and reuses nothing. One + * report measured stripping the line as taking the prefix cache from 0% to ~99.7%. + * https://github.com/anthropics/claude-code/issues/68900 + * https://github.com/anthropics/claude-code/issues/50085 + * + * Anthropic's own endpoint wants it intact, which is what the flag is for: it is on for the + * upstreams that treat the block as text and off for the one that reads it. What that endpoint + * does with it is not documented anywhere primary — the observable part is that the CLI adds + * *more* of these fields on its first-party path, so it reads as a client-attribution signal. */ export const stripBillingAttributionFromMessages = defineStage< Chat<'request.chat.messages' | 'route.attempt'>, @@ -896,6 +908,10 @@ export const stripBillingAttributionFromMessages = defineStage< }); const BILLING_HEADER_LINE = /x-anthropic-billing-header[^\n]*/g; +// Swept separately from the line because a `cch=` also reaches the prompt replayed inside +// message content, where the line pattern cannot see it. Every value captured in the wild is +// five hex digits with a trailing semicolon. +// https://github.com/anthropics/claude-code/issues/40652 const BILLING_CACHE_HASH = /cch=[0-9a-f]{5,};?/gi; const scrubText = (text: string): string => @@ -959,6 +975,12 @@ export const stripUnsupportedPartFieldsFromGemini = defineStage< })), }); +/** `fileData` addresses a Google-hosted file the API will not hand back — "you can use the API + * to get metadata about the files, but you can't download the files" — and the two code parts + * are the transcript of the server-side `codeExecution` tool that the stage below strips. + * None survives a translation, so they go at source. + * https://ai.google.dev/gemini-api/docs/files + * https://ai.google.dev/gemini-api/docs/code-execution */ const UNSUPPORTED_GEMINI_PART_FIELDS = ['fileData', 'executableCode', 'codeExecutionResult'] as const; const withUnsupportedPartFieldsStripped = (payload: GeminiPayload): GeminiPayload => { @@ -1016,6 +1038,13 @@ export const stripUnsupportedToolsFromGemini = defineStage< })), }); +/** Every field a Gemini tool group can carry other than `functionDeclarations` — the + * capabilities Google executes inside its own generation loop, which no target wire can be + * asked to run. `functionDeclarations` is the one that survives because the model "does not + * execute the function"; it asks the caller to. Re-derive against the `Tool` schema whenever + * it gains a field, which it still does. + * https://generativelanguage.googleapis.com/$discovery/rest?version=v1beta + * https://ai.google.dev/api/generate-content */ const UNSUPPORTED_GEMINI_TOOL_CAPABILITIES = [ 'googleSearch', 'googleSearchRetrieval', diff --git a/packages/gateway/src/data-plane/chat/messages/interceptors/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/messages/interceptors/answer-claude-code-probe.ts index 4f1f8d230e..b6a555d27d 100644 --- a/packages/gateway/src/data-plane/chat/messages/interceptors/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/messages/interceptors/answer-claude-code-probe.ts @@ -36,11 +36,16 @@ import { eventResult, providerModelOf, type ExecuteResult, type ModelCandidate } // serves still fails at the serve layer with a 404 and the CLI still reports // it not found. What we suppress is only the pointless one-token generation // behind it — no upstream call, and no tokens to bill. +// The CLI builds its agent string as +// `claude-cli/${VERSION} (external, ${CLAUDE_CODE_ENTRYPOINT ?? "cli"}…)`, read off the +// vendor's own shipped build — @anthropic-ai/claude-code-darwin-arm64@2.1.226, GIT_SHA +// e140b3281c1e8d834468889bd0a5c3fd2f15507c. +// https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-arm64/-/claude-code-darwin-arm64-2.1.226.tgz const CLAUDE_CODE_USER_AGENT = /^claude-cli\/\d+\.\d+\.\d+/i; -// The probes we can answer truthfully, written exactly as 2.1.226 sends them: -// `Hi` is the `/model` validation probe, `test` the credential check, which -// reads nothing at all off the response. +// The probes we can answer truthfully, read verbatim off that same build: +// `Hi` is the `model_validation` probe and `test` the `verify_api_key` +// credential check, which reads nothing at all off the response. // // Two recorded prompts are deliberately absent. `quota` is the rate-limit // preflight, and its caller consumes the `anthropic-ratelimit-unified-*` From 963fc52e41dfe92438e03f0b9f81ad46ec2a74c9 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 13:18:21 +0800 Subject: [PATCH 068/156] fix(chat): answer a client in its own protocol, and shape a turn for the wire it lands on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of the same rule, both found by review after the first pass fixed only the cases it happened to look at. A refusal crossing a translation kept the object it came in. That object is the *target* protocol's — an OpenAI `{error:{type,code}}` reaching a client reading Anthropic, or reaching Gemini, which has no wire of its own and so gets a foreign envelope on every refusal there is. A Gemini SDK reads `error.status`, which an OpenAI envelope has not got, and `error.code` as a number where it would find a string. The replaced surface forwarded an upstream body only when it was already Google-RPC shaped and otherwise wrote its own envelope around the upstream's words; that is what this restores, at the handoff, where the fact that a translation happened is known. A pair that does rewrite still carries its object across, because by then it is in the client's own protocol. The other half is the Chat Completions wire. `disableReasoningOnForcedToolChoice` and `stripPromptCacheKey` sat in the source chain, so a Messages, Responses or Gemini turn dialled over that wire got neither — the same defect fixed for the other two families, in the one file that fix did not touch. Both keep their place ahead of every vendor dialect: the canonical sentinel is emitted before a vendor spells it, and the field an upstream would reject is gone before a vendor rewrites what is left. Three tests asserted the behaviour being corrected here. They were written against the pipeline's own interim state rather than against the surface it replaces, so they are restated rather than deleted. --- .../data-plane/chat/gemini-pipeline_test.ts | 12 +++++--- .../data-plane/chat/routes-gemini_test.ts | 19 ++++++++---- .../data-plane/chat/translated-wires_test.ts | 30 ++++++++++++++++--- .../chat/chat-completions/pipeline.ts | 13 ++++---- .../gateway/src/data-plane/chat/handoff.ts | 14 ++++++++- 5 files changed, 68 insertions(+), 20 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index fe57895ba5..70e47a51ea 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -308,11 +308,15 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts()); expect(tried).toEqual(['up_a', 'up_b']); - // Every candidate failed, so the last failure is the base: the client sees the status an - // upstream actually returned and the words that upstream used, rather than a synthesized - // envelope quoting its serialized body back as a message. + // Every candidate failed, so the last failure is the base: the client sees the status the + // last upstream actually returned, and its words. The shape around them is this protocol's, + // because every Gemini wire is a translation and the object that came back was written by + // whatever protocol the candidate was dialled on. expect(facts['response.http.status']).toBe(400); - expect(facts['response.chat.gemini.rendered']).toEqual({ error: { message: 'no' } }); + const rendered = facts['response.chat.gemini.rendered'] as { error: { code: number; message: string; status: string } }; + expect(rendered.error.code).toBe(400); + expect(rendered.error.status).toBe('INVALID_ARGUMENT'); + expect(rendered.error.message).toContain('no'); // An upstream that was called and reported nothing, which is a different statement from // reporting zero. expect(facts['response.usage.billable']).toEqual([ diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts index 3f1dff2c51..bda4388984 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts @@ -119,7 +119,7 @@ test('a non-streaming turn reaches the client as one assembled Gemini body', asy expect(body.usageMetadata?.candidatesTokenCount).toBe(3); }); -test('an upstream refusal reaches the client in the upstream-s own words and status', async () => { +test('an upstream refusal reaches the client in this protocol-s own envelope', async () => { const { apiKey } = await setupAppTest(); const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota', code: 'quota_exceeded' } }; @@ -132,9 +132,16 @@ test('an upstream refusal reaches the client in the upstream-s own words and sta // The status the upstream sent, with none of the coercion a Google-RPC envelope would force // onto the codes it cannot express. expect(response.status).toBe(402); - // And the upstream's own object, rather than the same words quoted back inside - // `{ error: { code, message, status } }`. - expect(await response.json()).toEqual(refusal); + + // Gemini has no wire of its own, so every refusal that gets here was written by whatever + // protocol the candidate was dialled on — here OpenAI's. Handing that object on would answer + // a Gemini client in another protocol's words: `error.status` is the field its SDKs read and + // an OpenAI envelope has none, while `error.code` would arrive as a string where a number + // belongs. So the upstream's words become the message and this protocol writes the shape. + const body = await response.json() as { error: { code: number; message: string; status: string } }; + expect(body.error.code).toBe(402); + expect(typeof body.error.status).toBe('string'); + expect(body.error.message).toContain('copilot said no'); }); test('a refusal on a turn that asked to stream is a body, not an opened stream', async () => { @@ -152,7 +159,9 @@ test('a refusal on a turn that asked to stream is a body, not an opened stream', expect(response.status).toBe(402); expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); - expect(await response.json()).toEqual(refusal); + const body = await response.json() as { error: { code: number; message: string; status: string } }; + expect(body.error.code).toBe(402); + expect(body.error.message).toContain('copilot said no'); }); test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 829f3301bf..919c37b23c 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -397,9 +397,12 @@ describe('the fork over wires', () => { await drain(); }); - // A refusal a pair says nothing about reaches the client as the upstream wrote it, with the - // status the upstream sent. - it('hands a refusal the pair does not speak about up unchanged', async () => { + // A refusal a pair says nothing about keeps the status the upstream sent and loses the + // envelope it came in. The object was the *target* protocol's — an OpenAI `{error:{message}}` + // reaching a client that reads Anthropic — and answering one protocol in another's words is + // what the pair exists to prevent. What crosses is the status and the sentence; the client's + // own edge writes the shape. + it('carries a refusal the pair does not speak about without its foreign envelope', async () => { const callChatCompletions = vi.fn(async () => ({ ok: false as const, modelKey: 'k', response: Response.json({ error: { message: 'no' } }, { status: 403 }), })); @@ -408,7 +411,26 @@ describe('the fork over wires', () => { const { facts, drain } = await serveMessages(); expect(facts['response.http.status']).toBe(403); - expect(facts['response.chat.messages.rendered']).toEqual({ error: { message: 'no' } }); + const rendered = facts['response.chat.messages.rendered'] as { type: string; error: { type: string; message: string } }; + expect(rendered.type).toBe('error'); + expect(rendered.error.type).toBe('permission_error'); + // The upstream's own words survive as the sentence, which is what the replaced surface did + // with a body it could not forward. + expect(rendered.error.message).toContain('no'); + await drain(); + }); + + // The other half of the same rule: a rule that *can* speak carries the object across, because + // by then it is in the client's own protocol. + it('carries the envelope a pair did rewrite', async () => { + resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions: contextExceeded })]); + + const { facts, drain } = await serveMessages(); + + expect(facts['response.chat.messages.rendered']).toEqual({ + type: 'error', + error: { type: 'invalid_request_error', message: PROMPT_TOO_LONG_MESSAGE }, + }); await drain(); }); }); diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index 59eb0edc4f..fd716a7442 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -300,15 +300,18 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< * with the cache-bucket fold seeing cache fields under OpenAI's names and the carrier split * seeing usage the fold has already settled. * - * `disableReasoningOnForcedToolChoice` and `stripPromptCacheKey` sit above the fork rather - * than here, and both still precede every vendor dialect on the request path: the canonical - * sentinel is emitted before a vendor spells it, and the field an upstream would reject is - * gone before a vendor rewrites what is left. + * `disableReasoningOnForcedToolChoice` and `stripPromptCacheKey` are here rather than above + * the fork, because both speak about what an upstream's Chat Completions endpoint accepts — + * so a turn that arrived over a translation gets them too. Both still precede every vendor + * dialect on the request path: the canonical sentinel is emitted before a vendor spells it, + * and the field an upstream would reject is gone before a vendor rewrites what is left. */ export const chatCompletionsWire = (streamedUsage: string): readonly Stage[] => [ includeUsageStreamOptionsForChatCompletions, normalizeUsageForChatCompletions, + disableReasoningOnForcedToolChoiceForChatCompletions, applyRoleCompatibilityToChatCompletions, + stripPromptCacheKeyForChatCompletions, normalizeExclusiveCachedTokensForChatCompletions, vendorDeepSeekNormalizeForChatCompletions, vendorQwenNormalizeForChatCompletions, @@ -460,8 +463,6 @@ export const chatCompletionsServePipeline = (payload: ChatCompletionsPayload): P owns: [], }), materializeAttempt('request.chat.chatCompletions'), - disableReasoningOnForcedToolChoiceForChatCompletions, - stripPromptCacheKeyForChatCompletions, dialChatWire({ source: 'request.chat.chatCompletions', needs: ['request.chat.chatCompletions', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index e11547c6be..12c614cf19 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -138,7 +138,19 @@ export const handOff = [name, value])), body: new TextEncoder().encode(answer.message), }); - if (rewritten === undefined) return { ...up, [handoff.from.response]: answer } as never; + // A pair that declares no rewrite has nothing that can carry the refusal across, so what + // crosses is the status and the sentence and not the object they came in. The object is + // the *target* protocol's envelope — an OpenAI `{error:{type,code}}` where the client is + // reading Anthropic, or anything at all where the client is Gemini and reads + // `error.status` — and handing it on would answer one protocol in another's words. The + // family's own edge renders what is left, in the protocol its client actually speaks. + // + // Nothing is lost to the record: the target's failure was already written at the target's + // own key, with its body, before this stage consumed it. + if (rewritten === undefined) { + const { body: _foreign, envelope: _foreignEnvelope, ...carried } = answer; + return { ...up, [handoff.from.response]: move(carried) } as never; + } const text = new TextDecoder().decode(rewritten.body); let parsed: unknown; try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } From 19bae1de3aec7c8003b12fc637dd40d0c0a8ea1e Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 13:27:21 +0800 Subject: [PATCH 069/156] refactor(chat): state the gaps the chains actually have, and stop running four rules twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compaction chain ran four Responses wire rules above its ending, and both wires below re-run them. Idempotent, so nothing was wrong on the wire — but the run record showed each stage twice, and it was the wire-placement fix applied to one file and not its neighbour. Three comments were saying less than the truth. The Responses and Messages headers described their unported shims as "still only in the interceptor form", which reads as "runs by the old mechanism"; there is no interceptor array left, so it runs not at all, and a turn declaring a hosted tool reaches the upstream unshimmed. The handoff's refusal helper claimed assembly catches a family that misnames its streamed-usage key — assembly cannot, since the handoff is a wire's first stage and nothing above it in that composition needs the key; the runner's provides check on the dial is what catches it. And the shared error envelope explained its own test backwards. None of these changes behaviour except the de-duplication. What they change is whether the next reader is told the truth about where the gaps are. --- .../gateway/src/data-plane/chat/handoff.ts | 7 +++-- .../src/data-plane/chat/messages/pipeline.ts | 7 +++-- .../src/data-plane/chat/responses/compact.ts | 20 ++++++------- .../src/data-plane/chat/responses/pipeline.ts | 28 +++++++++++++------ .../protocols/src/common/error-envelope.ts | 17 +++++++---- 5 files changed, 51 insertions(+), 28 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index 12c614cf19..41e2dad81b 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -54,8 +54,11 @@ export interface Handoff>, use.gateway), + recordFrames( + wrapResponsesStatefulOutput(answer.frames as AsyncIterable>, use.gateway), + use.gateway.dump, + ), ); return { ...rest, @@ -381,10 +383,6 @@ export const responsesCompactPipeline = (payload: CanonicalResponsesPayload): Pi materializeAttempt('request.chat.responses'), beginStoredAttempt, expandShimCompactions, - disableReasoningOnForcedToolChoiceForResponses, - stripPromptCacheKeyForResponses, - vendorDeepSeekNormalizeForResponses, - vendorQwenNormalizeForResponses, dialResponsesCompaction, ]); }; diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index c180cb578d..a8e1fb8fac 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -32,10 +32,13 @@ // compaction routes and is rewritten exactly as a turn is. The dial here asks for // `generate`; what the ending answers with is the branch the provider says it ran, which // is why the envelope a compaction is arrives somewhere rather than nowhere. -// - this family's remaining interceptors, the server-tool shims among them. Still only in -// the interceptor form, so the array between the materialized payload and the fork is -// short rather than complete — and nothing in a pipelined turn writes to the store's -// private-payload scratchpad, because the shim that writes to it is what is missing. +// - the server-tool shims. They never became stages and the interceptor array that ran them +// is gone, so `withResponsesServerToolShim` and the two tools under it are reachable only +// from their own tests: a turn declaring a hosted `web_search` or `image_generation` tool +// reaches the upstream unshimmed, and the two flags that gate them do nothing. The code is +// kept so that porting it starts from something rather than from nothing. Nothing in a +// pipelined turn writes to the store's private-payload scratchpad for the same reason — +// the shim that writes to it does not run. // // One deliberate difference from `respond.ts`, shared with every other family on the // pipeline: an upstream that refused is answered in its own words, with the status it sent, @@ -57,6 +60,7 @@ import { normalizeAssistantInputText } from './items/normalize-assistant-content import { syntheticEventsFromResult } from './items/output.ts'; import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve-prep.ts'; import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; +import { recordFrames } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { AttemptSelector, BillableEntity } from '../../pipeline/facts.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; @@ -205,10 +209,18 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre // id beneath one response id this gateway minted, and the resource completed to what the // schema requires of it. It runs before the fold rather than beside it, so a client that // did not ask to stream is answered with the object the persisted frames add up to. - const frames = wrapResponsesClientEgress( - answer.frames as AsyncIterable>, - use.gateway, - client, + // + // The record is teed above all of it, which is what makes it a record of what the client + // was served rather than of what some layer below had still to rewrite. One tee covers + // every shape this edge can hand up, because all three read the same iterable: an SSE + // body, the events a transport frames itself, and the object the fold assembles. + const frames = recordFrames( + wrapResponsesClientEgress( + answer.frames as AsyncIterable>, + use.gateway, + client, + ), + use.gateway.dump, ); if (!back['ingress.chat.responses.wantsStream']) { try { diff --git a/packages/protocols/src/common/error-envelope.ts b/packages/protocols/src/common/error-envelope.ts index 73f3fe860f..04b74c73c9 100644 --- a/packages/protocols/src/common/error-envelope.ts +++ b/packages/protocols/src/common/error-envelope.ts @@ -1,11 +1,18 @@ // What a client is sent when a run produced no answer, and how an upstream's own words are // read out of what it sent. // -// An upstream that refused in its own words is handed on in them — a client reads the code -// and type inside, and the gateway has nothing truer to say than what the upstream said. This -// is why the test is whether an upstream answered at all rather than which shape it answered -// in: an OpenAI family's error is `{error:{...}}` and a rerank client's is `{message}`, and -// both are already the shape that client reads. +// An upstream that refused in its own words is handed on in them — a client reads the code and +// type inside, and the gateway has nothing truer to say than what the upstream said. The test +// is only that the upstream sent a JSON object, because the families that reach here differ in +// what that object looks like — an OpenAI family's error is `{error:{...}}` and a rerank +// client's is `{message}` — and a body that is not an object at all is one no JSON protocol's +// client can read, so it falls through to the envelope. +// +// The test is deliberately not "is this the client's own protocol". It cannot be, here: this +// function is shared and does not know which client is reading. Where a body can be the *wrong* +// protocol's — a translated wire, where the upstream spoke to a different protocol than the +// client did — the handoff drops it before it ever arrives, so what reaches here is only ever a +// body the client's own protocol produced. // // A refusal that never reached an upstream has only the gateway's own words, and those go in // the envelope every protocol here writes errors in. From 2d0a5f1393a2b8539d0b7ed5cb5ccb457c9511eb Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 13:31:57 +0800 Subject: [PATCH 070/156] refactor(chat): delete the onion the wires replaced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three modules the pipeline left behind, each with no caller but its own test. `traverseTranslation` threaded a translate trip around an inner attempt and invoked the pair's `apiError` hook on a refusal — which is `handOff`, stated as declarations instead of as a wrapper. `providerStreamResultToExecuteResult` turned a provider's stream into an `ExecuteResult`; each wire's ending reads one into facts now. `respondGemini` was that family's whole respond layer. Only one function survived any of them: the Google-RPC envelope this family's route answers an unknown action with, which `renderGeminiError` already writes. So the route builds it there, and nothing is left that needs a respond layer. The three tests go with them. What they covered is covered where the behaviour now lives — the pair's refusal rewrite in the translated-wire rows, and the Google-RPC envelope in the Gemini route rows. --- .../data-plane/chat/gemini/respond_test.ts | 95 -------- .../shared/provider-stream-result_test.ts | 87 ------- .../chat/shared/translate-traverse_test.ts | 160 ------------- .../src/data-plane/chat/gemini/http.ts | 11 +- .../src/data-plane/chat/gemini/respond.ts | 213 ------------------ .../chat/shared/provider-stream-result.ts | 59 ----- .../chat/shared/translate-traverse.ts | 30 --- 7 files changed, 7 insertions(+), 648 deletions(-) delete mode 100644 packages/gateway/__tests__/data-plane/chat/gemini/respond_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/shared/provider-stream-result_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/shared/translate-traverse_test.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini/respond.ts delete mode 100644 packages/gateway/src/data-plane/chat/shared/provider-stream-result.ts delete mode 100644 packages/gateway/src/data-plane/chat/shared/translate-traverse.ts diff --git a/packages/gateway/__tests__/data-plane/chat/gemini/respond_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini/respond_test.ts deleted file mode 100644 index 6869fa1841..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini/respond_test.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { Hono } from 'hono'; -import { test } from 'vitest'; - -import { respondGemini } from '../../../../src/data-plane/chat/gemini/respond.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventFrame } from '@floway-dev/protocols/common'; -import type { GeminiErrorResponse } from '@floway-dev/protocols/gemini'; -import type { ExecuteResult, InternalDebugError } from '@floway-dev/provider'; -import { assertEquals, assertExists } from '@floway-dev/test-utils'; - -const encoder = new TextEncoder(); - -const testTelemetryModelIdentity = { - model: 'test-model', - upstream: 'test-upstream', - modelKey: 'test-model-key', - pricing: null, -}; - -const ctx = () => mockChatGatewayCtx(); - -const requestGeminiResponse = async (result: ExecuteResult>): Promise => { - const app = new Hono(); - app.get('/', async c => await respondGemini(c, result, false, ctx())); - return await app.request('/'); -}; - -test('respondGemini preserves non-stream Gemini error event HTTP code', async () => { - const errorEvent: GeminiErrorResponse = { - error: { - code: 504, - status: 'DEADLINE_EXCEEDED', - message: 'timeout', - }, - }; - - const response = await requestGeminiResponse({ - type: 'events', - events: (async function* () { - yield eventFrame(errorEvent); - })(), - modelIdentity: testTelemetryModelIdentity, - }); - - assertEquals(response.status, 504); - assertEquals(await response.json(), errorEvent); -}); - -test('respondGemini preserves upstream Google RPC Status body', async () => { - const upstreamBody: GeminiErrorResponse = { - error: { - code: 412, - status: 'FAILED_PRECONDITION', - message: 'account is not ready', - }, - }; - - const response = await requestGeminiResponse({ - type: 'api-error', - source: 'upstream', - status: 400, - headers: new Headers({ 'content-type': 'application/json' }), - body: encoder.encode(JSON.stringify(upstreamBody)), - }); - - assertEquals(response.status, 412); - assertEquals(await response.json(), upstreamBody); -}); - -test('respondGemini internal errors include debug fields in Google RPC Status', async () => { - const error: InternalDebugError = { - type: 'internal_error', - name: 'TypeError', - message: 'boom', - stack: 'TypeError: boom\n at test', - cause: { upstream: 'bad shape' }, - target_api: 'responses', - }; - - const response = await requestGeminiResponse({ - type: 'internal-error', - status: 502, - error, - }); - const body = await response.json(); - - assertEquals(response.status, 502); - assertEquals(body.error.code, 502); - assertEquals(body.error.status, 'UNAVAILABLE'); - assertEquals(body.error.message, 'boom'); - assertEquals(body.error.stack, error.stack); - assertEquals(body.error.target_api, 'responses'); - assertExists(body.error.cause); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/provider-stream-result_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/provider-stream-result_test.ts deleted file mode 100644 index 6fe2b57d2a..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/shared/provider-stream-result_test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { describe, expect, test } from 'vitest'; - -import { providerStreamResultToExecuteResult } from '../../../../src/data-plane/chat/shared/provider-stream-result.ts'; -import { mockGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { stubModelCandidate } from '@floway-dev/test-utils'; - -const iter = (items: readonly T[]): AsyncIterable => ({ - async *[Symbol.asyncIterator]() { for (const item of items) yield item; }, -}); - -const okStreamResult = (events: AsyncIterable>): ProviderStreamResult => ({ - ok: true, - events, - modelKey: 'test-model-key', -}); - -const drainEvents = async (result: Awaited>>): Promise[]> => { - if (result.type !== 'events') throw new Error(`expected events result, got ${result.type}`); - const collected: ProtocolFrame[] = []; - for await (const frame of result.events) collected.push(frame); - return collected; -}; - -describe('providerStreamResultToExecuteResult (first-output-token stamping)', () => { - test('stamps firstOutputTokenAt on the first generated-token frame (messages thinking_delta)', async () => { - const ctx = mockGatewayCtx(); - const frames: ProtocolFrame[] = [ - { type: 'event', event: { type: 'message_start' } }, - { type: 'event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta', thinking: '...' } } }, - { type: 'event', event: { type: 'content_block_delta', delta: { type: 'text_delta', text: 'hi' } } }, - { type: 'event', event: { type: 'content_block_delta', delta: { type: 'text_delta', text: ' there' } } }, - ]; - const result = await providerStreamResultToExecuteResult(okStreamResult(iter(frames)), stubModelCandidate(), 'messages', ctx, () => null); - const collected = await drainEvents(result); - expect(collected).toEqual(frames); - expect(ctx.attempt.firstOutputTokenAt).not.toBe(null); - }); - - test('leaves firstOutputTokenAt null when only envelope frames appear', async () => { - const ctx = mockGatewayCtx(); - const frames: ProtocolFrame[] = [ - { type: 'event', event: { type: 'response.created' } }, - { type: 'event', event: { type: 'response.output_item.added' } }, - ]; - const result = await providerStreamResultToExecuteResult(okStreamResult(iter(frames)), stubModelCandidate(), 'responses', ctx, () => null); - await drainEvents(result); - expect(ctx.attempt.firstOutputTokenAt).toBe(null); - }); - - test('stamps at most once even for many output-content frames', async () => { - const ctx = mockGatewayCtx(); - const frames: ProtocolFrame[] = [ - { type: 'event', event: { choices: [{ delta: { content: 'a' } }] } }, - { type: 'event', event: { choices: [{ delta: { content: 'b' } }] } }, - { type: 'event', event: { choices: [{ delta: { content: 'c' } }] } }, - ]; - const result = await providerStreamResultToExecuteResult(okStreamResult(iter(frames)), stubModelCandidate(), 'chat-completions', ctx, () => null); - if (result.type !== 'events') throw new Error(`expected events result, got ${result.type}`); - const stampsAfterEachFrame: (number | null)[] = []; - for await (const _ of result.events) stampsAfterEachFrame.push(ctx.attempt.firstOutputTokenAt); - expect(stampsAfterEachFrame[0]).not.toBe(null); - // The subsequent frames must observe the exact same stamp — the stamping - // hook never overwrites once firstOutputTokenAt has been set. - expect(stampsAfterEachFrame[1]).toBe(stampsAfterEachFrame[0]); - expect(stampsAfterEachFrame[2]).toBe(stampsAfterEachFrame[0]); - }); -}); - -test('an abandoned stream still settles its metadata instead of hanging the caller', async () => { - // Every streaming response resolves its cost through finalMetadata, and the - // respond layer awaits it in a finally. A transport that walks away without - // closing the generator would hang that await forever. - const abort = new AbortController(); - const ctx = { ...mockGatewayCtx(), abortSignal: abort.signal }; - const frames: ProtocolFrame[] = [{ type: 'event', event: { type: 'response.created' } }]; - - const result = await providerStreamResultToExecuteResult(okStreamResult(iter(frames)), stubModelCandidate(), 'responses', ctx, () => null); - expect(result.type).toBe('events'); - if (result.type !== 'events') return; - - // Never iterate the events; just abandon them. - abort.abort(); - - expect((await result.finalMetadata!).modelIdentity).toBeDefined(); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/translate-traverse_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/translate-traverse_test.ts deleted file mode 100644 index f121beb580..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/shared/translate-traverse_test.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { describe, expect, test } from 'vitest'; - -import { traverseTranslation } from '../../../../src/data-plane/chat/shared/translate-traverse.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; - -// Fake event types — no real protocol types needed. -type SrcEvent = { kind: 'src'; value: string }; -type TgtEvent = { kind: 'tgt'; value: string }; - -const translate = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; -}> => ({ - target: `translated(${payload})`, - async *events(frames) { - for await (const frame of frames) { - if (frame.type === 'done') { - yield doneFrame(); - } else { - yield eventFrame({ kind: 'src', value: frame.event.value }); - } - } - }, -}); - -const makeTgtFrames = async function* (values: string[]): AsyncGenerator> { - for (const v of values) yield eventFrame({ kind: 'tgt', value: v }); - yield doneFrame(); -}; - -const collectFrames = async (iterable: AsyncIterable>): Promise[]> => { - const out: ProtocolFrame[] = []; - for await (const frame of iterable) out.push(frame); - return out; -}; - -const fakeModelIdentity = { modelId: 'fake' } as never; - -describe('traverseTranslation', () => { - test('events inner — outer wraps with trip.events', async () => { - const innerAttempt = async (_translated: string): Promise>> => ({ - type: 'events', - events: makeTgtFrames(['a', 'b']), - modelIdentity: fakeModelIdentity, - }); - - const result = await traverseTranslation('payload', translate, innerAttempt); - - expect(result.type).toBe('events'); - if (result.type !== 'events') throw new Error('unreachable'); - - const frames = await collectFrames(result.events); - expect(frames).toEqual([ - { type: 'event', event: { kind: 'src', value: 'a' } }, - { type: 'event', event: { kind: 'src', value: 'b' } }, - { type: 'done' }, - ]); - expect(result.modelIdentity).toBe(fakeModelIdentity); - }); - - test('result inner — outer returns the result unchanged', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 404, - headers: new Headers(), - body: new Uint8Array(), - }; - const innerAttempt = async (_translated: string): Promise>> => inner; - - const result = await traverseTranslation('payload', translate, innerAttempt); - - expect(result).toBe(inner); - }); - - test('failure inner — outer returns the failure unchanged', async () => { - const inner: ExecuteResult> = { - type: 'internal-error', - status: 500, - error: new Error('boom') as never, - }; - const innerAttempt = async (_translated: string): Promise>> => inner; - - const result = await traverseTranslation('payload', translate, innerAttempt); - - expect(result).toBe(inner); - }); - - test('upstream api-error with trip.apiError rewrite — outer returns the rewritten envelope', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 400, - headers: new Headers({ 'x-upstream-request-id': 'abc' }), - body: new TextEncoder().encode('raw upstream'), - }; - const translateWithRewrite = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; - apiError: (upstream: { status: number; headers: Headers; body: Uint8Array }) => { status: number; headers: Headers; body: Uint8Array }; - }> => ({ - ...(await translate(payload)), - apiError: () => ({ status: 400, headers: new Headers({ 'content-type': 'application/json' }), body: new TextEncoder().encode('rewritten') }), - }); - - const result = await traverseTranslation('payload', translateWithRewrite, async () => inner); - expect(result.type).toBe('api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - expect(new TextDecoder().decode(result.body)).toBe('rewritten'); - expect(result.headers.get('content-type')).toBe('application/json'); - expect(result.headers.get('x-upstream-request-id')).toBeNull(); - // Fields not touched by the rewriter (source, performance, upstream) survive. - expect(result.source).toBe('upstream'); - }); - - test('upstream api-error with trip.apiError returning undefined — outer passes the envelope through', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 502, - headers: new Headers(), - body: new TextEncoder().encode('bad gateway'), - }; - const translateWithRewrite = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; - apiError: () => undefined; - }> => ({ ...(await translate(payload)), apiError: () => undefined }); - - const result = await traverseTranslation('payload', translateWithRewrite, async () => inner); - expect(result).toBe(inner); - }); - - test('gateway-sourced api-error — trip.apiError is NOT invoked', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'gateway', - status: 400, - headers: new Headers(), - body: new TextEncoder().encode('synthesized'), - }; - let called = false; - const translateWithRewrite = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; - apiError: () => { status: number; headers: Headers; body: Uint8Array }; - }> => ({ - ...(await translate(payload)), - apiError: () => { - called = true; - return { status: 999, headers: new Headers(), body: new Uint8Array() }; - }, - }); - - const result = await traverseTranslation('payload', translateWithRewrite, async () => inner); - expect(called).toBe(false); - expect(result).toBe(inner); - }); -}); diff --git a/packages/gateway/src/data-plane/chat/gemini/http.ts b/packages/gateway/src/data-plane/chat/gemini/http.ts index d587fe5c62..7a86de043d 100644 --- a/packages/gateway/src/data-plane/chat/gemini/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini/http.ts @@ -9,7 +9,6 @@ import { geminiCountTokensPipeline } from './count-tokens.ts'; import { renderGeminiError } from './errors.ts'; import { geminiServePipeline } from './pipeline.ts'; -import { geminiRpcErrorResponse } from './respond.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; @@ -29,10 +28,14 @@ interface GeminiModelAction { // we split on the trailing `:` here so each entry sees just the action and // the resolved model id (with a leading `models/` prefix tolerated, as Google // SDKs send it). +/** A refusal about the route itself. It is answered before a model is resolved, so there is no + * attempt to record and nothing for a run to hold beyond the refusal. */ +const unknownAction = (message: string): Response => Response.json(renderGeminiError(404, message), { status: 404 }); + const parseGeminiModelAction = (modelAction: string | undefined): GeminiModelAction | Response => { - if (!modelAction) return geminiRpcErrorResponse(404, 'Missing Gemini model action.'); + if (!modelAction) return unknownAction('Missing Gemini model action.'); const separator = modelAction.lastIndexOf(':'); - if (separator <= 0 || separator === modelAction.length - 1) return geminiRpcErrorResponse(404, `Unknown Gemini model action: ${modelAction}`); + if (separator <= 0 || separator === modelAction.length - 1) return unknownAction(`Unknown Gemini model action: ${modelAction}`); return { model: modelAction.slice(0, separator).replace(/^models\//, ''), action: modelAction.slice(separator + 1) }; }; @@ -55,7 +58,7 @@ export const geminiHttp = async (c: AuthedContext): Promise => { if (parsed.action === 'generateContent' || parsed.action === 'streamGenerateContent') { return await runGeminiGenerate(c, parsed.model, parsed.action === 'streamGenerateContent'); } - return geminiRpcErrorResponse(404, `Unknown Gemini model action: ${parsed.action}`); + return unknownAction(`Unknown Gemini model action: ${parsed.action}`); }; /** A body the gateway could not read is reported in the words this protocol's own clients diff --git a/packages/gateway/src/data-plane/chat/gemini/respond.ts b/packages/gateway/src/data-plane/chat/gemini/respond.ts deleted file mode 100644 index d181e6d5a4..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini/respond.ts +++ /dev/null @@ -1,213 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; -import { geminiStatusForHttpStatus } from './errors.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import { eventFrame, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; -import { geminiProtocolFrameToSSEFrame, GEMINI_MISSING_TERMINAL_MESSAGE, isGeminiErrorEvent, isGeminiTerminalEvent, collectGeminiProtocolEventsToResult } from '@floway-dev/protocols/gemini'; -import type { GeminiErrorResponse, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import { type ExecuteResult, type PlainResult, type ApiErrorResult, type InternalDebugError, toInternalDebugError, decodeApiErrorBody } from '@floway-dev/provider'; - -// Renders an upstream Gemini result into the client HTTP/SSE response, in the -// Google-RPC error envelope. An error-typed result is a pre-stream failure and -// always answers as HTTP; an events result drains to one JSON body -// (non-streaming) or is proxied frame by frame (streaming). -export const respondGemini = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - ctx: GatewayCtx, -): Promise => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return geminiApiErrorResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return geminiErrorResponse(result.status, result.error.message, internalDebugFields(result.error)); - } - - if (result.type === 'plain') { - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); - } - - const state = new SourceStreamState(); - const observed = observeGeminiFrames(result.events, state, ctx); - const frames = wrapGeminiAffinityEgress(observed, affinityEgressOptions(ctx)); - - if (!wantsStream) { - try { - const response = await collectGeminiProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return geminiCollectErrorResponse(error); - } - } - - forwardUpstreamHeaders(c, result.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, geminiSseFrames(frames, state, ctx), { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`gemini stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); -}; - -// --- error rendering: Google-RPC envelope --- - -type GeminiErrorDebugFields = Partial> & { target_api?: string }; - -type GeminiErrorStatusPayload = { - error: GeminiErrorResponse['error'] & GeminiErrorDebugFields; -}; - -const googleRpcHttpStatusCode = (status: number): number => (Number.isInteger(status) && status >= 400 && status <= 599 ? status : 500); - -const geminiRpcErrorPayload = (status: number, message: string, debug: GeminiErrorDebugFields = {}): GeminiErrorStatusPayload => { - const code = googleRpcHttpStatusCode(status); - return { - error: { code, message, status: geminiStatusForHttpStatus(code), ...debug }, - }; -}; - -const internalDebugFields = (error: InternalDebugError): GeminiErrorDebugFields => ({ - type: error.type, - name: error.name, - stack: error.stack, - cause: error.cause, - ...(error.target_api ? { target_api: error.target_api } : {}), -}); - -const geminiInternalRpcErrorPayload = (status: number, error: unknown): GeminiErrorStatusPayload => { - const debug = toInternalDebugError(error); - return geminiRpcErrorPayload(status, debug.message, internalDebugFields(debug)); -}; - -// Response builders. The count_tokens path under `http.ts` reuses them -// alongside the error renderer for its synthesized JSON envelope. -export const geminiRpcErrorResponse = (status: number, message: string): Response => { - const payload = geminiRpcErrorPayload(status, message); - return Response.json(payload, { status: payload.error.code }); -}; - -export const geminiInternalRpcErrorResponse = (status: number, error: unknown): Response => { - const payload = geminiInternalRpcErrorPayload(status, error); - return Response.json(payload, { status: payload.error.code }); -}; - -const geminiErrorResponse = (status: number, message: string, debug: GeminiErrorDebugFields = {}): Response => { - // For gateway-minted errors, a non-500 that maps to INTERNAL is coerced to 500. - const code = geminiStatusForHttpStatus(status) === 'INTERNAL' && status !== 500 ? 500 : status; - return Response.json({ error: { code, message, status: geminiStatusForHttpStatus(code), ...debug } }, { status: code }); -}; - -const geminiApiErrorResponse = (error: ApiErrorResult): Response => googleRpcErrorPassthroughResponse(error) ?? geminiErrorResponse(error.status, apiErrorMessage(error)); - -const geminiCollectErrorResponse = (error: unknown): Response => { - const geminiError = caughtGeminiErrorEvent(error); - return geminiError ? Response.json(geminiError, { status: googleRpcHttpStatusCode(geminiError.error.code) }) : geminiInternalRpcErrorResponse(502, error); -}; - -// Recognizing / extracting an upstream-shaped Gemini error from a raw body or a -// thrown cause, so a native Google error is forwarded verbatim rather than -// re-wrapped. -const parseJson = (value: string): unknown => { - try { - return JSON.parse(value); - } catch { - return undefined; - } -}; - -const isGeminiErrorResponse = (value: unknown): value is GeminiErrorResponse => { - if (!value || typeof value !== 'object' || !('error' in value)) return false; - const error = (value as { error?: unknown }).error; - if (!error || typeof error !== 'object') return false; - const payload = error as Partial; - return typeof payload.code === 'number' && typeof payload.message === 'string' && typeof payload.status === 'string'; -}; - -const googleRpcErrorPassthroughResponse = (error: ApiErrorResult): Response | null => { - const parsed = parseJson(decodeApiErrorBody(error).trim()); - if (!isGeminiErrorResponse(parsed)) return null; - - return new Response(error.body.slice(), { - status: googleRpcHttpStatusCode(parsed.error.code), - headers: new Headers(error.headers), - }); -}; - -const apiErrorMessage = (error: ApiErrorResult): string => { - const body = decodeApiErrorBody(error).trim(); - return body || 'Upstream Gemini request failed.'; -}; - -const caughtGeminiErrorEvent = (error: unknown): GeminiErrorResponse | null => { - if (!(error instanceof Error)) return null; - return isGeminiErrorResponse(error.cause) ? error.cause : null; -}; - -const geminiStreamErrorEvent = (error: unknown): GeminiStreamEvent => - (caughtGeminiErrorEvent(error) ?? geminiInternalRpcErrorPayload(500, error)) as unknown as GeminiStreamEvent; - -// --- frame observation --- - -const isGeminiTerminalFrame = (frame: ProtocolFrame): boolean => frame.type === 'done' || (frame.type === 'event' && isGeminiTerminalEvent(frame.event)); - -const observeGeminiFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - for await (const frame of frames) { - ctx.dump?.frame(frame); - const failed = frame.type === 'event' && isGeminiErrorEvent(frame.event); - if (failed) state.failed = true; - if (isGeminiTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isGeminiTerminalFrame(frame)) return; - } - throw new Error(GEMINI_MISSING_TERMINAL_MESSAGE); -}; - -const geminiSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - try { - for await (const frame of frames) { - const sse = geminiProtocolFrameToSSEFrame(frame); - if (sse) yield sse; - } - } catch (error) { - state.failed = true; - const event = geminiStreamErrorEvent(error); - ctx.dump?.frame(eventFrame(event)); - yield sseFrame(JSON.stringify(event)); - } -}; diff --git a/packages/gateway/src/data-plane/chat/shared/provider-stream-result.ts b/packages/gateway/src/data-plane/chat/shared/provider-stream-result.ts deleted file mode 100644 index 3c8e0e3980..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/provider-stream-result.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { isFirstOutputTokenFrame } from './first-output-token.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import type { BillableUsage, ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventResult, readUpstreamApiError, type ChatTargetApi, type EventResultMetadata, type ExecuteResult, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; - -export const providerStreamResultToExecuteResult = async ( - providerResult: ProviderStreamResult, - candidate: ModelCandidate, - targetApi: ChatTargetApi, - ctx: GatewayCtx, - // Reads the upstream's own usage off one of its events, in the upstream's - // own protocol. This is the only place pricing figures are produced; nothing - // downstream re-derives them from the translated result the client receives. - readBillableUsage: (event: TEvent) => BillableUsage | null, -): Promise>> => { - const context = upstreamPerformanceContext(ctx, candidate, 'chat'); - if (!providerResult.ok) { - return { ...(await readUpstreamApiError(providerResult.response, candidate.provider.upstreamId)), performance: context }; - } - const identity = telemetryModelIdentity(candidate, providerResult.modelKey); - let resolveFinal!: (metadata: EventResultMetadata) => void; - const finalMetadata = new Promise(resolve => { resolveFinal = resolve; }); - // Only a report carrying real counts replaces the running figure, so a - // trailing empty usage frame cannot wipe a good one. Held outside the - // generator so an abandoned stream can still settle with what it saw. - let billableUsage: BillableUsage | undefined; - const settleMetadata = (): void => resolveFinal({ - modelIdentity: identity, - ...(context !== undefined ? { performance: context } : {}), - ...(billableUsage !== undefined ? { billableUsage } : {}), - }); - // Every streaming response now resolves its cost here, and the respond - // layer awaits it in a `finally`. A transport that walks away without - // closing the generator would otherwise hang that await forever, so the - // abort settles it too; whichever fires first wins, and the later call is a - // no-op. - ctx.abortSignal?.addEventListener('abort', settleMetadata, { once: true }); - const stampedEvents = (async function* () { - try { - for await (const frame of providerResult.events) { - if (ctx.attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, targetApi)) { - ctx.attempt.firstOutputTokenAt = performance.now(); - } - if (frame.type === 'event') { - const reported = readBillableUsage(frame.event); - if (reported !== null) billableUsage = reported; - } - yield frame; - } - } finally { - settleMetadata(); - } - })(); - return { - ...eventResult(stampedEvents, identity, { performance: context, headers: providerResult.headers }), - finalMetadata, - }; -}; diff --git a/packages/gateway/src/data-plane/chat/shared/translate-traverse.ts b/packages/gateway/src/data-plane/chat/shared/translate-traverse.ts deleted file mode 100644 index a647ebb624..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/translate-traverse.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import type { TranslateTripResult } from '@floway-dev/translate'; - -// Threads a translate trip around an inner attempt. The trip itself is async -// (the real `@floway-dev/translate` pair functions resolve a `Promise`), so -// `translate` returns the trip object behind a promise. The pair functions -// take `(src, ctx)`; this helper's `translate` parameter stays unary so each -// caller closes over its own `ctx` (`p => translateXViaY(p, ctx)`). -// -// On an upstream api-error the trip's optional `apiError` hook is invoked so -// the pair can rewrite the body into the source protocol's envelope — the -// canonical case is `messages-via-*` translating an upstream context-window -// error into the Anthropic `prompt is too long:` shape Claude Code recognizes -// for auto-compaction. Pairs that don't set `apiError` (or return `undefined`) -// pass the upstream body through verbatim. -export const traverseTranslation = async ( - payload: SP, - translate: (p: SP) => Promise>, - innerAttempt: (translated: TP) => Promise>>, -): Promise>> => { - const trip = await translate(payload); - const inner = await innerAttempt(trip.target); - if (inner.type === 'events') return { ...inner, events: trip.events(inner.events) }; - if (inner.type === 'api-error' && inner.source === 'upstream' && trip.apiError !== undefined) { - const rewritten = trip.apiError({ status: inner.status, headers: inner.headers, body: inner.body }); - if (rewritten !== undefined) return { ...inner, status: rewritten.status, headers: rewritten.headers, body: rewritten.body }; - } - return inner; -}; From b727a311316637d6c538742352254e4038bfa9c4 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 13:32:35 +0800 Subject: [PATCH 071/156] docs(chat): say the prologue's rule once Two adjacent comments made overlapping claims about the same two lines. What matters is the one that has already been violated: building the options twice takes the body twice, and the second read is empty. --- packages/gateway/src/data-plane/chat/prologue.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts index d0c3f58d3d..f1bfd7909c 100644 --- a/packages/gateway/src/data-plane/chat/prologue.ts +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -33,11 +33,10 @@ export const openChatPrologue = ( readonly storeFactory: (apiKey: ApiKey, requestStartedAt: number) => StatefulResponsesStore; }, ): ChatPrologue => { - // One context, built once: the body is read once and `takeRequestBody` empties what it was - // given, so the chat context is what this run *has* rather than a second one over it. // One options object, read twice: the context is built from it, and the run recording it - // opened is what the runner's events are written to. Building the options again would take - // the body a second time and hand the dump an empty buffer. + // opened is what the runner's events are written to. Building it a second time would call + // `takeRequestBody` again and hand the dump an empty buffer — which is exactly how every + // chat turn once came to write a record holding no events. const ctxOptions = gatewayCtxOptions(c, ingress, options); const gateway = createChatGatewayCtxFromHono(c, ctxOptions, options.storeFactory); const base = prologueFor(gateway, ingress, runDumpOf(ctxOptions)); From 9c7a43c241e0c55868f388e07ab0b2adf19a49c3 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 13:35:50 +0800 Subject: [PATCH 072/156] fix(responses): put the frames a client was served back in the record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run record is read for what the client got, and the dashboard's collected view is assembled by replaying exactly those frames. Three chat edges tee them into the record; the two Responses edges did not, so `/v1/responses` and `/v1/responses/compact` recorded stage boundaries and no content at all — the shape of a turn with none of it. The replaced surface recorded them from `respond.ts`. The tee goes at the edge, above the client-egress wrapper, because that is the last place the frames are this protocol's own and already carry everything every layer of the egress rewrites into them. One tee covers both shapes the generate edge hands out — the SSE body and the object the fold assembles — because both read the same iterable, which is also why a non-streaming turn records. Not the events framing. The WebSocket transport reads those frames and writes each one itself, and it already records what it wrote; teeing here as well would enter every frame twice and make the replay show the turn twice. That is the division `recordSentPayloadBytes` already draws for the bytes, so it is drawn the same way for the frames, and the transport's count is now fenced by a test. The compaction edge tees above the stateful half, where the frames carry the ids its resource is assembled under. They never reach a client there, so the record is the only place they survive at all. --- .../chat/responses/websocket_test.ts | 42 ++++- .../data-plane/chat/run-dump-frames_test.ts | 171 ++++++++++++++---- .../src/data-plane/chat/responses/pipeline.ts | 26 +-- 3 files changed, 191 insertions(+), 48 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts index e23cdb57e2..96947cb298 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts @@ -7,7 +7,7 @@ import { KEEP_ALIVE_EVENT_TYPE } from '../../../../src/data-plane/chat/responses import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../../../src/data-plane/shared/sse.ts'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../../src/repo/usage-metrics.ts'; -import { installDumpStubs, runRecordOf } from '../../../dump/test-fixtures.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../../dump/test-fixtures.ts'; import { FakeTime } from '../../../test-time.ts'; import { buildCodexUpstreamRecord, codexModels, copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseResponsesResponse } from '../../../test-utils/app.ts'; import { installWorkerWebSocketRuntime, type TestWorkerWebSocket } from '../../../test-utils/worker-websocket.ts'; @@ -299,6 +299,46 @@ test('Responses WebSocket dump responseBytes equals the UTF-8 payload bytes sent ); }); +/** How many frames a run record holds. The format writes each recorded frame as a + * `stream.frame` event, so counting them reads no object table. */ +const recordedFrameCount = (record: Parameters[0]): number => + eventsOf(runRecordOf(record)) + .filter(event => event.type === 'stream.frame') + .reduce((total, event) => total + (event.frames as readonly unknown[]).length, 0); + +// The record holds one frame per event the turn sent, and the count is what says so. Two +// readers sit on the same iterable — the family's edge, which tees what a client is served +// into the record, and this transport, which reads it to write each event to the socket — so a +// second tee is invisible everywhere except here, and it would make the collected view the +// dashboard replays show the whole turn twice. +test('Responses WebSocket records one frame per event it sent', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + + await withSuccessfulResponsesUpstream( + async () => await withWorkerWebSocketRuntime(async () => { + const client = await connectResponsesWebSocket(apiKey.key); + const recorded = recordRawMessages(client); + try { + await completeResponsesTurn(client, 'one-frame-per-event'); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + + // A keep-alive is the transport's own idle write rather than a frame of the turn, so + // it is not one of the events the record is counted against. + const sent = recorded.messages + .map(message => JSON.parse(message) as { type?: unknown }) + .filter(message => message.type !== KEEP_ALIVE_EVENT_TYPE); + assert(sent.length > 0, 'expected the turn to have sent events at all'); + assertEquals(recordedFrameCount(dumps.stored[0]?.record), sent.length); + } finally { + recorded.stop(); + client.close(); + } + }), + ); +}); + // A turn on this transport is a run, and a run settles once: the chain reads what the // upstream metered off its own events as they pass, and the transport settles that reading // after the last frame has gone out. The rows are filtered to this turn's own key because diff --git a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts index 0679642c8d..7734bedea3 100644 --- a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts @@ -4,15 +4,27 @@ // its content, and the dashboard's collected view is assembled by replaying exactly these // events. The tee sits at the family's edge, so what lands in the record is the source // protocol's frames — what the client read — rather than whatever the upstream spoke. +// +// Every edge that hands a client frames is covered here, in both shapes the client can be +// handed them: the stream itself, and the one body the same frames were folded into. +// `/v1/responses/compact` is one of those edges rather than a wire under the generate chain — +// it reads its frames itself, so recording them is its own to do — and it only ever folds, +// because a compaction is one resource. import { test, vi } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; -import { copilotModels, flushAsyncWork, requestApp, setupAppTest, sseChatCompletionsResponse } from '../../test-utils/app.ts'; +import { copilotModels, flushAsyncWork, requestApp, setupAppTest, sseChatCompletionsResponse, sseResponsesResponse } from '../../test-utils/app.ts'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; -const withChatCompletionsUpstream = async (run: () => Promise): Promise => +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer each turn decides for itself. */ +const withCopilot = async ( + models: Parameters[0], + serve: (request: Request) => Promise | Response, + run: () => Promise, +): Promise => await withMockedFetch( async request => { const url = new URL(request.url); @@ -20,19 +32,8 @@ const withChatCompletionsUpstream = async (run: () => Promise): Promise if (url.pathname === '/copilot_internal/v2/token') { return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); } - if (url.pathname === '/models') { - return jsonResponse(copilotModels([{ id: 'gpt-frames', supported_endpoints: ['/chat/completions'] }])); - } - if (url.pathname === '/chat/completions') { - return sseChatCompletionsResponse({ - id: 'chatcmpl_frames', - object: 'chat.completion', - model: 'gpt-frames', - choices: [{ index: 0, message: { role: 'assistant', content: 'hello there' }, finish_reason: 'stop' }], - usage: { prompt_tokens: 3, completion_tokens: 5, total_tokens: 8 }, - }); - } - throw new Error(`Unhandled fetch ${request.url}`); + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return await serve(request); }, run, ); @@ -59,22 +60,19 @@ const resolveFrames = (events: readonly Record[]): unknown[] => .flatMap(event => (event.frames as unknown[]).map(resolve)); }; -const recordedFramesOf = async (stream: boolean): Promise => { +/** Runs one turn with a record open, and hands back the frames that record holds. + * + * Reading the client's body is part of the turn rather than an assertion about it: reading is + * what records, so a caller that never read would prove the opposite of what these tests + * claim. */ +const framesRecordedBy = async (turn: (apiKey: string) => Promise): Promise => { const { apiKey, repo } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); - await withChatCompletionsUpstream(async () => { - const response = await requestApp('/v1/chat/completions', { - method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify({ model: 'gpt-frames', messages: [{ role: 'user', content: 'hi' }], stream }), - }); - assertEquals(response.status, 200); - // Reading is what records, so a test that never read the body would prove the opposite of - // what it claims. - await response.text(); - }); + const response = await turn(apiKey.key); + assertEquals(response.status, 200); + await response.text(); await flushAsyncWork(); await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); @@ -83,18 +81,121 @@ const recordedFramesOf = async (stream: boolean): Promise => return resolveFrames(eventsOf(runRecordOf(stored.record))); }; -test('a streamed chat turn records the frames it served', async () => { - const frames = await recordedFramesOf(true); +/** The content is the point, not the count: a record holding frames with nothing in them is + * the same empty record with more events. */ +const assertFramesCarry = (frames: readonly unknown[], text: string): void => { assertEquals(frames.length > 0, true); - // The content is the point, not the count: a record holding frames with nothing in them is - // the same empty record with more events. - assertEquals(JSON.stringify(frames).includes('hello there'), true); + assertEquals(JSON.stringify(frames).includes(text), true); +}; + +/** The output items the recorded frames closed, which is what a resource assembled from them + * is made of. */ +const closedItemTypes = (frames: readonly unknown[]): readonly string[] => + frames.flatMap(frame => { + const { event } = frame as { event?: { type?: string; item?: { type?: string } } }; + if (event?.type !== 'response.output_item.done') return []; + return event.item?.type === undefined ? [] : [event.item.type]; + }); + +const ANSWER = 'hello there'; + +const chatCompletionsTurn = (stream: boolean) => async (apiKey: string): Promise => + await withCopilot( + [{ id: 'gpt-frames', supported_endpoints: ['/chat/completions'] }], + () => sseChatCompletionsResponse({ + id: 'chatcmpl_frames', + object: 'chat.completion', + model: 'gpt-frames', + choices: [{ index: 0, message: { role: 'assistant', content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 3, completion_tokens: 5, total_tokens: 8 }, + }), + async () => await requestApp('/v1/chat/completions', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ model: 'gpt-frames', messages: [{ role: 'user', content: 'hi' }], stream }), + }), + ); + +const responsesTurn = (stream: boolean) => async (apiKey: string): Promise => + await withCopilot( + [{ id: 'gpt-responses-frames', supported_endpoints: ['/responses'] }], + () => sseResponsesResponse({ + id: 'resp_frames', + object: 'response', + model: 'gpt-responses-frames', + status: 'completed', + output: [{ + type: 'message', id: 'msg_frames', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text: ANSWER, annotations: [] }], + }], + error: null, + incomplete_details: null, + usage: { input_tokens: 11, output_tokens: 7, total_tokens: 18 }, + }), + async () => await requestApp('/v1/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ + model: 'gpt-responses-frames', + input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }], + stream, + }), + }), + ); + +/** The history a compaction is asked to stand in for. Copilot's compaction retains the + * messages it did not absorb, so this is what the resource carries beside the blob. */ +const HISTORY = 'everything that has been said so far'; + +const compactionTurn = async (apiKey: string): Promise => + await withCopilot( + [{ id: 'gpt-compact-frames', supported_endpoints: ['/responses'] }], + // Copilot has no compaction endpoint of its own: it replays the protocol over `/responses` + // with the trigger appended and `stream: false`, so the answer here is one JSON body. + () => jsonResponse({ + id: 'resp_compaction', + object: 'response', + model: 'gpt-compact-frames', + status: 'completed', + output: [{ type: 'compaction', id: 'cmp_frames', encrypted_content: 'OPAQUE_COMPACTION_BLOB' }], + error: null, + incomplete_details: null, + usage: { input_tokens: 21, output_tokens: 4, total_tokens: 25 }, + }), + async () => await requestApp('/v1/responses/compact', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ + model: 'gpt-compact-frames', + input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: HISTORY }] }], + }), + }), + ); + +test('a streamed chat turn records the frames it served', async () => { + assertFramesCarry(await framesRecordedBy(chatCompletionsTurn(true)), ANSWER); }); test('a collected chat turn records the frames it was assembled from', async () => { // 「流式请求前端看到流+前端组装结果,非流式请求前端看到流+原始组装结果」 — the frames are // recorded either way, which is what lets the operator see the stream behind a single body. - const frames = await recordedFramesOf(false); - assertEquals(frames.length > 0, true); - assertEquals(JSON.stringify(frames).includes('hello there'), true); + assertFramesCarry(await framesRecordedBy(chatCompletionsTurn(false)), ANSWER); +}); + +test('a streamed responses turn records the frames it served', async () => { + assertFramesCarry(await framesRecordedBy(responsesTurn(true)), ANSWER); +}); + +test('a collected responses turn records the frames it was assembled from', async () => { + assertFramesCarry(await framesRecordedBy(responsesTurn(false)), ANSWER); +}); + +test('a compaction records the frames its resource was assembled from', async () => { + const frames = await framesRecordedBy(compactionTurn); + // These frames never reached a client — the client got the one resource they add up to — so + // the record is the only place the turn's own events survive at all. Both halves of what + // that resource was assembled from are in them: the history the upstream retained, and the + // compaction item the next turn inherits. + assertFramesCarry(frames, HISTORY); + assertEquals(closedItemTypes(frames).includes('compaction'), true); }); diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index a8e1fb8fac..79212b26f3 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -209,19 +209,21 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre // id beneath one response id this gateway minted, and the resource completed to what the // schema requires of it. It runs before the fold rather than beside it, so a client that // did not ask to stream is answered with the object the persisted frames add up to. - // - // The record is teed above all of it, which is what makes it a record of what the client - // was served rather than of what some layer below had still to rewrite. One tee covers - // every shape this edge can hand up, because all three read the same iterable: an SSE - // body, the events a transport frames itself, and the object the fold assembles. - const frames = recordFrames( - wrapResponsesClientEgress( - answer.frames as AsyncIterable>, - use.gateway, - client, - ), - use.gateway.dump, + const egress = wrapResponsesClientEgress( + answer.frames as AsyncIterable>, + use.gateway, + client, ); + // The record is teed above all of that, which is what makes it a record of what the client + // was served rather than of what some layer below had still to rewrite. One tee covers both + // shapes this edge hands out itself, because both read the same iterable: the SSE body, and + // the object the fold assembles from the frames that would have gone out. + // + // The events framing is not one of them. There the frames go to a transport that writes + // each one itself, and a transport that writes its own frames records them — the same + // division `recordSentPayloadBytes` draws for the bytes, and for the same reason: only the + // transport knows which of them reached the socket. + const frames = framing === 'events' ? egress : recordFrames(egress, use.gateway.dump); if (!back['ingress.chat.responses.wantsStream']) { try { return { From 1a1404a75c3bf8754276c447e46d4f8f9759f590 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 13:41:44 +0800 Subject: [PATCH 073/156] fix(gemini): refuse an unmeasurable body the way the same body is refused on generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `:countTokens` reaches Messages through a pair of its own rather than a handoff, because what crosses the boundary is one measurement and not frames. The translator it runs is the same one, and it refuses input it would otherwise have to coerce or silently drop — but nothing caught that refusal here, so it escaped the run: a body carrying a `functionResponse` part in model content answered 500 with an internal-error envelope and a stack trace, naming the gateway as the party at fault for a request that was simply not measurable. Generation was fixed at the handoff; the replaced surface answered both actions alike, in Gemini's own Google-RPC envelope with a 400. So the pair answers it, exactly as the handoff does: a status and a sentence, no envelope, because the edge above renders a failure in this protocol's own words and is the only place that knows what a Gemini client reads. Answering rather than throwing also makes the verdict a candidate's rather than the run's. The three keys it answers with are what "no upstream was measured" looks like on a counting chain — the refusal, nothing billed, no upstream headers. There is no streamed reading among them, and that is the whole difference from the generating chains: a measurement never opens a stream, so this chain carries no such key for a refusal to have to settle. --- .../chat/translation-refusal_test.ts | 55 ++++++++++++++----- .../data-plane/chat/gemini/count-tokens.ts | 41 +++++++++++--- 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts index 28919d8c60..6b89b364df 100644 --- a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts @@ -9,6 +9,11 @@ // It is answered rather than thrown for a second reason the replaced surface had no way to // express: the verdict belongs to one candidate. The next candidate may be reachable on a wire // whose protocol *can* carry the body, and failover re-runs the suffix to find out. +// +// Both of Gemini's actions cross a protocol boundary and both are covered: generation reaches +// its target through a handoff, and `:countTokens` through the pair that asks the question in +// Messages. The replaced surface answered a refusal the same way on either, because what the +// caller reads is the same protocol whichever action it asked for. import { test } from 'vitest'; @@ -25,8 +30,8 @@ const upstream = (models: Parameters[0]) => async (request throw new Error(`Unhandled fetch ${request.url}`); }; -// A `functionResponse` part in model content: a shape Gemini defines and Chat Completions has -// nowhere to put, so the translator refuses it rather than dropping it. +// A `functionResponse` part in model content: a shape Gemini defines and neither target protocol +// has anywhere to put, so the translator refuses it rather than dropping it. const untranslatableGeminiBody = { contents: [ { role: 'model', parts: [{ functionResponse: { name: 'f', response: { ok: true } } }] }, @@ -34,27 +39,49 @@ const untranslatableGeminiBody = { ], }; +/** What a Gemini client reads a refusal out of. `error.status` is the field it keys on, and it + * is not a field the gateway's internal-error envelope has — nor is `stack` a field a + * caller-input refusal has any business carrying. */ +const assertGoogleRpcRefusal = async (response: Response): Promise => { + assertEquals(response.status, 400); + const body = await response.json() as { error: { code: number; status: string; message: string } }; + assertEquals(body.error.code, 400); + assertEquals(body.error.status, 'INVALID_ARGUMENT'); + assertEquals(body.error.message.includes('not supported in model content'), true); + assertEquals('stack' in body.error, false); +}; + test('a body the target protocol cannot carry is refused in the client own protocol', async () => { const { apiKey } = await setupAppTest(); await withMockedFetch( upstream([{ id: 'gpt-translate', supported_endpoints: ['/chat/completions'] }]), async () => { - const response = await requestApp('/v1beta/models/gpt-translate:generateContent', { + await assertGoogleRpcRefusal(await requestApp('/v1beta/models/gpt-translate:generateContent', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify(untranslatableGeminiBody), + })); + }, + ); +}); + +// Counting crosses the same boundary. It has no handoff — what crosses is one measurement, not +// frames — so the refusal it can hit is its own to answer, and answering it anywhere else means +// a caller asking what a turn would cost is told the gateway broke. +test('a body the counting wire cannot carry is refused in the client own protocol', async () => { + const { apiKey } = await setupAppTest(); + + await withMockedFetch( + // Counting is reachable only over an upstream's own Messages endpoint, so this is the one + // candidate shape that gets as far as the translation. + upstream([{ id: 'claude-count', supported_endpoints: ['/messages'] }]), + async () => { + await assertGoogleRpcRefusal(await requestApp('/v1beta/models/claude-count:countTokens', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify(untranslatableGeminiBody), - }); - - // Google-RPC, not the gateway's internal-error envelope: `error.status` is the field a - // Gemini client reads, and a 500 would name the gateway as the party at fault. - assertEquals(response.status, 400); - const body = await response.json() as { error: { code: number; status: string; message: string } }; - assertEquals(body.error.code, 400); - assertEquals(body.error.status, 'INVALID_ARGUMENT'); - assertEquals(body.error.message.includes('not supported in model content'), true); - // The stack trace an internal failure carries has no business in a caller-input refusal. - assertEquals('stack' in body.error, false); + })); }, ); }); diff --git a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts index b9a58197fa..20f55b26b5 100644 --- a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts @@ -16,7 +16,9 @@ // // The pair is not `handOff`. A handoff maps frames, and there are none here: what crosses the // protocol boundary is one measurement, and reading Anthropic's counts back out as Google's -// `{ totalTokens }` is the whole of the mapping. +// `{ totalTokens }` is the whole of the mapping. What it does share with a handoff is that it +// runs a translator, so it answers what the translator refuses — the same refusal, and the same +// 400, whichever action a client asked this protocol for. // // The strippers run for the reason they run on generation: what no translation can carry // cannot be sent, so it goes at source. The thought suppressor does not — it rewrites a @@ -40,7 +42,7 @@ import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import type { GeminiPayload } from '@floway-dev/protocols/gemini'; -import { translateGeminiViaMessages } from '@floway-dev/translate'; +import { translateGeminiViaMessages, TranslatorInputError } from '@floway-dev/translate'; /** Counting is reachable only over an upstream's own Messages endpoint: no other protocol * answers the question, and no translation invents an answer. */ @@ -103,12 +105,16 @@ const emitGeminiTokenCount = defineStage< * It consumes this protocol's request key and provides the target's, exactly as a handoff * does, so the wire below sees only Messages and cannot tell it was reached by translation. * What differs is the way back: there are no frames to map, only counts. + * + * It carries the `return` trait for the same reason a handoff does — a body the target + * protocol cannot represent is answered here rather than measured. */ const measureGeminiAsMessages = defineStage< G<'request.chat.gemini' | 'route.attempt'>, G<'request.chat.messages'>, Counted<'response.chat.messages'>, Counted<'response.chat.gemini'>, + Counted<'response.chat.gemini'> & G<'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'measureGeminiAsMessages', @@ -124,17 +130,38 @@ const measureGeminiAsMessages = defineStage< provides: ['response.chat.gemini'], }, }, + return: { provides: ['response.chat.gemini', 'response.usage.billable', 'response.http.headers'] }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - const trip = await translateGeminiViaMessages(facts['request.chat.gemini'], { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - }); + const { 'request.chat.gemini': asked, ...down } = facts; + + let trip; + try { + trip = await translateGeminiViaMessages(asked, { + model: candidate.model.id, + fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, + }); + } catch (error) { + // Input the counting wire's protocol cannot represent is the client's fault, and the + // translator is what knows which part of the body was at fault. Answering rather than + // throwing is also what keeps it a *candidate's* verdict, so the fork can try the next + // one. Anything else raised here is a fault of this gateway's and rides up as one. + if (!(error instanceof TranslatorInputError)) throw error; + return move({ + ...down, + 'response.chat.gemini': { status: 400, message: error.message }, + // Nothing was measured, which is what an empty billed set and an empty header list say + // on the other two keys. There is no streamed reading to leave behind: a measurement + // never opens a stream, which is why this chain carries no such key at all. + 'response.usage.billable': [], + 'response.http.headers': [], + }); + } + // The translator writes a turn, and a turn says how its answer is delivered. There is no // answer to deliver here, so the field does not travel. const { stream: _stream, ...target } = trip.target; - const { 'request.chat.gemini': _asked, ...down } = facts; const back = await next({ ...down, 'request.chat.messages': move(target) }); const { 'response.chat.messages': counted, ...up } = back; return { ...up, 'response.chat.gemini': move(asGeminiTokenCount(counted)) }; From af919a5b2920dfc30eb93e855ddd88a94b04084b Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 13:47:15 +0800 Subject: [PATCH 074/156] refactor(responses): record a turn's frames once, wherever it is framed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The edge teed the frames into the record for the SSE transport and stood down for the WebSocket one, which teed its own — so the family's edge branched on how a transport frames, for a question that is not about framing. The stated reason was that only a transport knows which frames reached the socket, but the WebSocket tee recorded on the way *in* to its writer, not after it, so the two recorded the same frames and only the placement differed. One tee, at the edge, for both. A WebSocket turn is the same frames rendered differently, which is the rule the whole design rests on, and reading is what records — so a transport that stopped early still records exactly what it took. The test that made this safe to collapse is the one that counts: recorded frames against the messages the socket actually sent. Re-adding the second tee fails it. --- .../gateway/src/data-plane/chat/responses/pipeline.ts | 10 +++++----- .../gateway/src/data-plane/chat/responses/websocket.ts | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index 79212b26f3..daf31baaea 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -219,11 +219,11 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre // shapes this edge hands out itself, because both read the same iterable: the SSE body, and // the object the fold assembles from the frames that would have gone out. // - // The events framing is not one of them. There the frames go to a transport that writes - // each one itself, and a transport that writes its own frames records them — the same - // division `recordSentPayloadBytes` draws for the bytes, and for the same reason: only the - // transport knows which of them reached the socket. - const frames = framing === 'events' ? egress : recordFrames(egress, use.gateway.dump); + // Both transports too. A WebSocket turn is the same frames rendered differently, so it is + // recorded here rather than where it is framed — one tee for the family, whatever writes + // what it hands up. Reading is what records, so a transport that stopped early records + // exactly what it took. + const frames = recordFrames(egress, use.gateway.dump); if (!back['ingress.chat.responses.wantsStream']) { try { return { diff --git a/packages/gateway/src/data-plane/chat/responses/websocket.ts b/packages/gateway/src/data-plane/chat/responses/websocket.ts index dd587e1f22..8b26bdfbb2 100644 --- a/packages/gateway/src/data-plane/chat/responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/responses/websocket.ts @@ -459,7 +459,7 @@ const respondResponsesWebSocket = async (input: { let completion: StreamCompletion = 'error'; try { let terminalEvent: ClientResponsesStreamEvent | undefined; - const iterator = observeResponsesWebSocketFrames(rendered, state, ctx)[Symbol.asyncIterator](); + const iterator = observeResponsesWebSocketFrames(rendered, state)[Symbol.asyncIterator](); let pendingNext = pendingWsFrameResult(iterator.next()); let completed = false; let stoppedByDownstream = false; @@ -644,13 +644,13 @@ const respondResponsesWebSocket = async (input: { } }; +/** Reads the turn's own ending off the frames on their way to the socket. The record is not + * its business: the edge tees these same frames, so what reaches here is already recorded. */ const observeResponsesWebSocketFrames = async function* ( frames: AsyncIterable>, state: SourceStreamState, - ctx: ChatGatewayCtx, ): AsyncGenerator> { for await (const frame of frames) { - ctx.dump?.frame(frame); if (frame.type === 'event') { const event = frame.event; const failed = event.type === 'error' || event.type === 'response.failed'; From 9b1a20ad697e1f88324078ec1214b4afda70b555 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 14:31:16 +0800 Subject: [PATCH 075/156] docs(chat): make every chain say what it is, and drop what nothing reaches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verification pass found that the fixes had left their own comments behind. Five chain headers listed rows their compositions no longer have or omitted rows they gained: the Chat Completions header still named an interceptor array, the compaction header a set of rules moved into its wires, both count-token headers left out the settlement one of them was fixed to run, and the Messages header never listed the probe that answers without dialling. Worse, three comments documented the defects the fixes removed. The Gemini header presented an upstream's own error object reaching a Gemini client as a deliberate choice — it is the thing that made that client unreadable, and the handoff now drops it. The handoff's own JSDoc said a pair with no rewrite hands a refusal up unchanged, seventy lines above the code that strips it. Three more places said the server-tool shim is "still an interceptor", which reads as runs-by-the-old-mechanism rather than does-not-run. Six exports went the way of the three deleted earlier, for the same reason: nothing but their own tests reached them. `SourceStreamState` was the one thing left alive in `shared/respond.ts`, so it moves to a file named for it — the Responses socket needs it to decide whether what it wrote was a finished turn, and no HTTP family does. One test asserted `typeof error.status === 'string'`, which any string passes. It is the field a Google SDK branches on, so it asserts the value now. --- .../data-plane/chat/routes-gemini_test.ts | 4 +- ...nd_test.ts => source-stream-state_test.ts} | 2 +- .../chat/chat-completions/pipeline.ts | 2 +- packages/gateway/src/data-plane/chat/facts.ts | 3 -- .../data-plane/chat/gemini/count-tokens.ts | 1 + .../src/data-plane/chat/gemini/pipeline.ts | 16 +++++--- .../gateway/src/data-plane/chat/handoff.ts | 4 +- .../data-plane/chat/messages/count-tokens.ts | 6 ++- .../chat/messages/interceptors/types.ts | 11 +----- .../src/data-plane/chat/messages/pipeline.ts | 1 + .../src/data-plane/chat/responses/compact.ts | 7 ++-- .../chat/responses/interceptors/types.ts | 24 +----------- .../data-plane/chat/responses/items/store.ts | 8 ++-- .../src/data-plane/chat/responses/pipeline.ts | 6 +-- .../data-plane/chat/responses/websocket.ts | 2 +- .../src/data-plane/chat/shared/respond.ts | 38 ------------------- .../chat/shared/source-stream-state.ts | 16 ++++++++ 17 files changed, 54 insertions(+), 97 deletions(-) rename packages/gateway/__tests__/data-plane/chat/shared/{respond_test.ts => source-stream-state_test.ts} (96%) delete mode 100644 packages/gateway/src/data-plane/chat/shared/respond.ts create mode 100644 packages/gateway/src/data-plane/chat/shared/source-stream-state.ts diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts index bda4388984..2940820a08 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts @@ -140,7 +140,9 @@ test('an upstream refusal reaches the client in this protocol-s own envelope', a // belongs. So the upstream's words become the message and this protocol writes the shape. const body = await response.json() as { error: { code: number; message: string; status: string } }; expect(body.error.code).toBe(402); - expect(typeof body.error.status).toBe('string'); + // The Google-RPC name for the status, which is the field an SDK branches on. A code this + // envelope cannot express reads `INTERNAL`, and 402 is one of them. + expect(body.error.status).toBe('INTERNAL'); expect(body.error.message).toContain('copilot said no'); }); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/respond_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/source-stream-state_test.ts similarity index 96% rename from packages/gateway/__tests__/data-plane/chat/shared/respond_test.ts rename to packages/gateway/__tests__/data-plane/chat/shared/source-stream-state_test.ts index ec3a3dc54a..ec19efd66b 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/respond_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/source-stream-state_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { SourceStreamState } from '../../../../src/data-plane/chat/shared/respond.ts'; +import { SourceStreamState } from '../../../../src/data-plane/chat/shared/source-stream-state.ts'; import { assertEquals } from '@floway-dev/test-utils'; // ── SourceStreamState classification ── diff --git a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts index fd716a7442..db2cf725b6 100644 --- a/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/chat-completions/pipeline.ts @@ -4,7 +4,7 @@ // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate -// the interceptors the shared array, each one a stage +// materializeAttempt puts the payload this candidate is owed into the record // dialChatWire the ending: picks this candidate's wire and hands into it // // The wire a candidate is dialled on is chosen per candidate rather than at assembly, and diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts index ca22f89e8a..87b1da0db7 100644 --- a/packages/gateway/src/data-plane/chat/facts.ts +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -21,9 +21,6 @@ import type { ResponsesPayload } from '@floway-dev/protocols/responses'; * and no target role. */ export type ChatSourceProtocol = 'chatCompletions' | 'messages' | 'responses' | 'gemini'; -/** The three a translation can target. */ -export type ChatTargetProtocol = Exclude; - export interface ChatFacts extends GatewayFacts { /** Which protocol the client spoke. Read by the stages that must know — a vendor * normalizer genuinely does — rather than reached for through an ambient field. diff --git a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts index 20f55b26b5..7ad0899ebd 100644 --- a/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini/count-tokens.ts @@ -1,6 +1,7 @@ // Gemini's `:countTokens` as a pipeline. // // emitGeminiTokenCount the edge: writes the measurement, or the refusal, as one body +// writeSettlement above the fork, so a measurement is sampled once however many it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record diff --git a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts index 0ca220dbe8..549520d313 100644 --- a/packages/gateway/src/data-plane/chat/gemini/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini/pipeline.ts @@ -36,12 +36,16 @@ // ended without a terminal event. The chain detects one and leaves the run as a throw // rather than as that 502 envelope. // -// One deliberate difference from `respond.ts`, shared with every other family on the -// pipeline: an upstream that refused is answered in its own words. A Chat Completions wire -// refuses in the OpenAI envelope, and that object is handed on as it came rather than being -// quoted back inside a Google-RPC `{ error: { code, message, status } }` — which also means -// the status the upstream sent is the status the client sees, with no coercion of the codes -// that envelope maps to `INTERNAL`. +// A refusal is the one place where having no wire of its own changes what a client is owed. +// Every other family can hand an upstream's own error object on, because the upstream spoke +// that family's protocol. Here it never did: whatever came back was written by Chat +// Completions, Messages or Responses, and `error.status` — the field a Google client reads — +// is in none of them. So the words survive and the shape is this protocol's, which is what +// the handoff arranges by dropping a foreign envelope on the way up. +// +// The status is still the upstream's, with no coercion of the codes a Google-RPC envelope +// maps to `INTERNAL`. That part is a departure from the replaced surface, which raised such a +// status to 500. import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; import { analyzeGeminiAffinity } from './affinity/ingress.ts'; diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index 41e2dad81b..64d56c3bff 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -75,7 +75,9 @@ const refuseTranslation = (response: ResponseKey, message: string): Record; - -// Payload-only transforms can run in both the streaming generation chain and -// the one-shot count_tokens chain. Keeping the result generic prevents those -// shared interceptors from accidentally inspecting either result shape. -export type MessagesPayloadInterceptor = ( - ctx: MessagesInvocation, - gatewayCtx: GatewayCtx, - run: InterceptorRun, -) => Promise; diff --git a/packages/gateway/src/data-plane/chat/messages/pipeline.ts b/packages/gateway/src/data-plane/chat/messages/pipeline.ts index dba9a38dce..24420da575 100644 --- a/packages/gateway/src/data-plane/chat/messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/messages/pipeline.ts @@ -5,6 +5,7 @@ // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record +// answerClaudeCodeProbe answers the CLI's model probe without dialling anything // dialChatWire the ending: picks this candidate's wire and hands into it // // Three wires, all handing up `response.chat.messages`: this protocol's own, and the two diff --git a/packages/gateway/src/data-plane/chat/responses/compact.ts b/packages/gateway/src/data-plane/chat/responses/compact.ts index d44c3e406e..8242f100e5 100644 --- a/packages/gateway/src/data-plane/chat/responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/responses/compact.ts @@ -8,7 +8,6 @@ // materializeAttempt puts the payload this candidate is owed into the record // beginStoredAttempt reseeds the store's per-attempt scratchpad // expandShimCompactions a compaction this gateway wrote, back into what it stood for -// this family's request rules, the same four generation runs above its fork // dialResponsesCompaction the ending: dials a compaction, or simulates one // // A second **operation** over this protocol rather than another wire under @@ -36,9 +35,9 @@ // What the simulation cannot reproduce is stated where it happens, at `summarizationTurnFor`. // // One thing this chain does not carry, stated rather than implied by its absence: the -// server-tool shim. It is still only an interceptor, so a compaction whose caller declared a -// server tool is summarized without the ReAct loop wrapped around it — the same gap the -// generate chain states, and for the same reason. +// server-tool shim. It never became a stage and the interceptor array that ran it is gone, so +// a compaction whose caller declared a server tool is summarized without the ReAct loop wrapped +// around it — the same gap the generate chain states, and for the same reason. import { responsesCreatedAt, wrapResponsesStatefulOutput } from './client-output.ts'; import { completeResponsesCompaction } from './compaction-resource.ts'; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/types.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/types.ts index 34aa8d86b2..88eb70c54b 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/types.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/types.ts @@ -1,31 +1,11 @@ -import type { TokenUsage } from '../../../../repo/types.ts'; import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; import type { Interceptor } from '@floway-dev/interceptor'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import type { EventResultMetadata, ExecuteResult, ResponsesInvocation, TelemetryModelIdentity } from '@floway-dev/provider'; +import type { ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { ExecuteResult, ResponsesInvocation } from '@floway-dev/provider'; export type { ResponsesInvocation }; -// The chain runner produces an event stream for both actions — the attempt -// drains it into a single non-streaming result when the caller's intent action -// was 'compact', and serve completes that result into the compaction resource. -// `modelIdentity`, `usage`, and `performance` carry the per-turn attribution -// forward so the http layer records the success path identically to streaming -// generate. -// The non-streaming result is parameterized because `/responses/compact` -// answers with `CompactResource` rather than the response resource, and its -// own completion narrows the type further. -export type ResponsesAttemptResult = - | ExecuteResult> - | { - readonly type: 'result'; - readonly result: Result; - readonly modelIdentity: TelemetryModelIdentity; - readonly usage: TokenUsage | null; - readonly performance: EventResultMetadata['performance']; - }; - export type ResponsesInterceptor = Interceptor< ResponsesInvocation, ChatGatewayCtx, diff --git a/packages/gateway/src/data-plane/chat/responses/items/store.ts b/packages/gateway/src/data-plane/chat/responses/items/store.ts index f370998b86..690b3701a4 100644 --- a/packages/gateway/src/data-plane/chat/responses/items/store.ts +++ b/packages/gateway/src/data-plane/chat/responses/items/store.ts @@ -389,10 +389,10 @@ export const createResponsesHttpStore = (apiKey: ResponsesStatePolicy, requestSt // Non-Responses sources (Messages / Gemini / Chat Completions) never persist // Responses items, even when translation enters a Responses attempt — but the -// server-tool shim still runs there, and its request-private payload -// scratchpad lives on the store. So they get a store with no backing: it holds -// per-attempt state in memory and reads/writes nothing durable, keeping the -// store present on every chat ctx. +// server-tool shim's request-private payload scratchpad lives on the store, and +// it is what will write there once that shim is a stage. So they get a store +// with no backing: it holds per-attempt state in memory and reads/writes nothing +// durable, keeping the store present on every chat ctx. export const createNonResponsesSourceStore = (apiKeyId: string): StatefulResponsesStore => new LayeredStatefulResponsesStore({ apiKeyId, reads: [], writes: [] }); diff --git a/packages/gateway/src/data-plane/chat/responses/pipeline.ts b/packages/gateway/src/data-plane/chat/responses/pipeline.ts index daf31baaea..88724218b2 100644 --- a/packages/gateway/src/data-plane/chat/responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/responses/pipeline.ts @@ -765,9 +765,9 @@ export const hydrateStoredItems = ( * * Below the fork because it is per attempt: what one attempt wrote into the scratchpad is * not what the next one starts from, and re-running the suffix is what clears it. Nothing in - * a pipelined turn writes to it yet — the server-tool shim is what writes, and it is still - * an interceptor — so today this only puts back the state the stored rows already carried, - * which is what lets an item this turn re-emits be stored with it intact. + * a pipelined turn writes to it — the server-tool shim is what writes, and it does not run at + * all — so today this only puts back the state the stored rows already carried, which is what + * lets an item this turn re-emits be stored with it intact. */ export const beginStoredAttempt = defineStage< R<'request.chat.responses.privatePayloads'>, diff --git a/packages/gateway/src/data-plane/chat/responses/websocket.ts b/packages/gateway/src/data-plane/chat/responses/websocket.ts index 8b26bdfbb2..a107467f67 100644 --- a/packages/gateway/src/data-plane/chat/responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/responses/websocket.ts @@ -25,7 +25,7 @@ import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS, type StreamCompletion } from '../../ import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; import type { ChatPrologue } from '../prologue.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { SourceStreamState } from '../shared/respond.ts'; +import { SourceStreamState } from '../shared/source-stream-state.ts'; import { move, run } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; diff --git a/packages/gateway/src/data-plane/chat/shared/respond.ts b/packages/gateway/src/data-plane/chat/shared/respond.ts deleted file mode 100644 index 284010379b..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/respond.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { StreamCompletion } from '../../shared/sse.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { plainResult } from '@floway-dev/provider'; -import type { EventResultMetadata, ExecuteResult, PlainResult } from '@floway-dev/provider'; - -// Emits a measurement endpoint's already-shaped body verbatim. The endpoint's -// `attempt` owns all shaping — the success body and any source-specific error -// envelope — so every source's `respond` renders a plain result identically. -export const plainResultToResponse = (result: PlainResult): Response => - new Response(result.body.slice().buffer, { status: result.status, headers: result.headers }); - -// Used by count_tokens endpoints that either pass through the upstream body -// or wrap an already-built error/success Response. -export const plainResultFromResponse = async (response: Response, upstream?: string): Promise => - plainResult( - response.status, - new Headers({ 'content-type': response.headers.get('content-type') ?? 'application/json' }), - new Uint8Array(await response.arrayBuffer()), - upstream, - ); - -// Per-stream observation accumulated by each source's frame observer and read -// back when the response settles: did the stream fail, did it reach its -// terminal frame, and the last frame-level usage worth billing. -export class SourceStreamState { - failed = false; - completed = false; - - failedAfter(completion: StreamCompletion): boolean { - return completion === 'error' || this.failed || (completion === 'cancel' && !this.completed); - } -} - -export const eventResultMetadata = async (result: Extract>, { type: 'events' }>): Promise => - await (result.finalMetadata ?? { - modelIdentity: result.modelIdentity, - ...(result.performance ? { performance: result.performance } : {}), - }); diff --git a/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts b/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts new file mode 100644 index 0000000000..bb481f309f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts @@ -0,0 +1,16 @@ +// What a source's stream turned out to be, read back when the turn settles. +// +// One transport still needs it: the Responses WebSocket writes each frame itself and has to +// decide, when the socket closes, whether what it wrote was a finished turn. Every HTTP +// family answers the same question from its own metered reading instead. + +import type { StreamCompletion } from '../../shared/sse.ts'; + +export class SourceStreamState { + failed = false; + completed = false; + + failedAfter(completion: StreamCompletion): boolean { + return completion === 'error' || this.failed || (completion === 'cancel' && !this.completed); + } +} From cb82a0124ef5ac36349ae676fa9403636d2c9216 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 14:36:59 +0800 Subject: [PATCH 076/156] docs(chat): stop claiming an interceptor array that is gone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four comments argued that migrating to stages did not delete the guards `ctx.targetApi` carried, and grounded that in "there is one interceptor array". There is not, and the argument survives without it — the guards were split rather than kept in one place. A guard about a *wire* is now said by position, because a stage below the fork runs only on the wire the fork chose. A guard about what the *client* spoke is said by needing `ingress.chat.sourceProtocol`, which is the one that survives a protocol switch. The count-token chain justified its rule ordering by "the order the interceptor chain ran them", which is provenance for a mechanism no longer in the tree. It runs them in the order generation runs them, which is the property that has to hold and the one a reader can check. --- packages/gateway/src/data-plane/chat/facts.ts | 11 +++++------ packages/gateway/src/data-plane/chat/interceptors.ts | 5 +++-- .../src/data-plane/chat/messages/count-tokens.ts | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts index 87b1da0db7..5f76cc8ca6 100644 --- a/packages/gateway/src/data-plane/chat/facts.ts +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -5,8 +5,8 @@ // its own request key and its own response key, so a translation *consumes* the source and // *provides* the target, and which protocol a stage is looking at is a declared need rather // than an ambient field on a mutable context. That is visibility and checkability, not the -// deletion of the guards themselves — with one interceptor array the guards remain, and the -// migration is an equivalent one. And `ingress.*` is what the client asked for: it survives +// deletion of the guards themselves — each is still made, by position where it is about a wire +// and by this key where it is about what the client spoke. And `ingress.*` is what the client asked for: it survives // the switch to whatever protocol the upstream turned out to speak, which is why // `wantsStream` is four disjoint keys rather than one shared one. @@ -25,10 +25,9 @@ export interface ChatFacts extends GatewayFacts { /** Which protocol the client spoke. Read by the stages that must know — a vendor * normalizer genuinely does — rather than reached for through an ambient field. * - * This is what `ctx.targetApi` was, read through `needs` rather than reached for. The - * guards that field carried are not deleted by saying it this way — there is one - * interceptor array, so a stage that must not run on a re-entered request still says so - * itself. What changed is that the thing it tests is declared. */ + * This is what `ctx.targetApi` was, read through `needs` rather than reached for. A stage + * that must not run on a request which re-entered its protocol still says so itself, by + * needing this key. What changed is that the thing it tests is declared. */ 'ingress.chat.sourceProtocol': ChatSourceProtocol; /** What the client asked for, per protocol and disjoint, because a translated request * must not inherit the target protocol's answer to a question the client never asked. */ diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index ed56c6a5b9..5287a8342c 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -12,8 +12,9 @@ // is frozen and `delete ctx.payload.x` would throw. `ctx.targetApi` is a declared need rather // than an ambient field — array position says which chain a stage is in, and what it reads // about the protocol it is looking at is in the record. The guards themselves are not deleted -// by that: there is one interceptor array, so a rule that must not run on a re-entered request -// still says so. And a rule that reads the response — the two that rewrite a stream — states +// by that, they are split: a guard about the *wire* is said by position, because a stage below +// the fork runs only on the wire the fork chose, and a guard about what the *client* spoke is +// said by needing `ingress.chat.sourceProtocol`, which survives the switch. And a rule that reads the response — the two that rewrite a stream — states // it as a response-direction declaration rather than as work after `await run()`, with what it // learned on the way down carried in the stage's own closure, because a response-side `needs` // can only name what the ending provides. diff --git a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts index 6304ab6cdb..8af8278d07 100644 --- a/packages/gateway/src/data-plane/chat/messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/messages/count-tokens.ts @@ -23,9 +23,9 @@ // too, and sits with it. // // What it measures is what generation would send. The same three request rules run, in the -// order the interceptor chain ran them, and the web-search shim's request half runs above -// them: a client that asked for the native server tool is measured on the client-tool shape -// the gateway would actually put on the wire. +// order generation runs them, and the web-search shim's request half runs above them: a client +// that asked for the native server tool is measured on the client-tool shape the gateway would +// actually put on the wire. import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { renderMessagesError } from './errors.ts'; From 2b44e280e6979be3e225a9811cbb68e55f855e52 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 05:25:12 +0800 Subject: [PATCH 077/156] fix: resolve the compact-shim conflict left by the nested-path removal --- .../chat/openai-responses/compact-shim.ts | 116 ------------------ 1 file changed, 116 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts index f0fd9a00ba..ee87ba3f01 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts @@ -1,7 +1,6 @@ // Compact-shim — what a `response.compaction` envelope is made of, where no upstream wire // makes one. // -<<<<<<<< HEAD:packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts // The rules that use these are stages: `simulatesCompaction` says which candidates' // compactions are the shim's to make, `expandShimCompactions` puts a blob of ours back into // the items it stood for, and `summarizeForCompaction` sends the turn built here and packs @@ -9,21 +8,6 @@ // because a compaction the shim wrote is issued through one entry — `/v1/responses/compact`, // or a generate turn whose input ends in a `compaction_trigger` — and echoed back into the // other by the client that received it. -======== -// Engagement is the OR of two conditions: -// 1. The per-upstream `responses-compact-shim` flag is on. This is the -// operator-controlled opt-in for OpenAI-Responses-target upstreams that -// already answer a compact request themselves — natively through -// `/responses/compact` (codex / azure / custom), or by replaying -// `RemoteCompactionV2` over `/responses` (copilot) — but where we still -// want shim-synthesized envelopes. -// 2. The candidate's `targetApi` is not `responses`. When the upstream is -// Anthropic Messages or OpenAI Chat Completions, the translation layer has no concept -// of a `compaction` output item or a `compaction_trigger` input item. -// The shim is structurally required regardless of the flag — without -// it, a `compaction_trigger` item would reach the translator and -// crash on the unknown variant. ->>>>>>>> feat/pipeline-non-chat:packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts // // What is here is the shim's own substance: // @@ -48,20 +32,9 @@ // selectively turn the flag off for the codex / copilot / azure / custom upstreams that // answer compact themselves. -<<<<<<<< HEAD:packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../shared/base64url-json.ts'; import { isJsonObject } from '../../../shared/json-helpers.ts'; import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesOutputItem, ResponsesResult } from '@floway-dev/protocols/responses'; -======== -import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from './types.ts'; -import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../shared/base64url-json.ts'; -import { isJsonObject } from '../../../../shared/json-helpers.ts'; -import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; -import { syntheticEventsFromResult } from '../items/output.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectOpenAIResponsesProtocolEventsToResult, createRandomOpenAIResponsesItemId, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesInputItem, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf, type ExecuteResult } from '@floway-dev/provider'; ->>>>>>>> feat/pipeline-non-chat:packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts // The two vendored constants below (SUMMARIZATION_PROMPT and SUMMARY_PREFIX) // are the compactor system prompt and the handoff prefix openai/codex ships @@ -184,21 +157,12 @@ export const expandShimCompactionItems = (payload: CanonicalOpenAIResponsesPaylo // ── Outbound summarization ──────────────────────────────────────────────────── -<<<<<<<< HEAD:packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts -======== -type ChainRun = () => Promise>>; - ->>>>>>>> feat/pipeline-non-chat:packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts // The spec makes the item lifecycle the authority and requires nothing of the // terminal's `output`; a Codex upstream states an `output` that omits the // assistant message it just closed. A turn that closed nothing falls back to // the terminal, as the client-facing egress does. // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L237 -<<<<<<<< HEAD:packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts export const summaryTextFrom = (closed: Map, stated: readonly ResponsesOutputItem[]): string => { -======== -const summaryTextFrom = (closed: Map, stated: readonly OpenAIResponsesOutputItem[]): string => { ->>>>>>>> feat/pipeline-non-chat:packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts const items = closed.size === 0 ? stated : [...closed].sort(([left], [right]) => left - right).map(([, item]) => item); @@ -212,11 +176,7 @@ const summaryTextFrom = (closed: Map, stated: return parts.join(''); }; -<<<<<<<< HEAD:packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts export const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: ResponsesResult): ResponsesResult => { -======== -const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: OpenAIResponsesResult): OpenAIResponsesResult => { ->>>>>>>> feat/pipeline-non-chat:packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts // The prefix lives inside the blob so it round-trips atomically with the // summary — a downstream LLM sees `${SUMMARY_PREFIX}\n${summaryText}` in // one message and reads it as "another LLM's handoff", not as the human @@ -256,18 +216,12 @@ const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: O }; }; -<<<<<<<< HEAD:packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts // The turn a compaction is simulated with: the compactor's own prompt, the // history that is being compacted, and a nudge to produce the summary now. // Exported because the pipeline's compaction chain sends the same turn — one // definition is what keeps the simulated compaction identical whichever entry // asked for it. export const summarizationTurnFor = (payload: CanonicalResponsesPayload): CanonicalResponsesPayload => { -======== -const simulateCompaction = async (ctx: OpenAIResponsesInvocation, gatewayCtx: ChatGatewayCtx, run: ChainRun): Promise>> => { - const originalPayload = ctx.payload; - ->>>>>>>> feat/pipeline-non-chat:packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts // Strip compaction_trigger so the upstream sees a plain generate turn // against SUMMARIZATION_PROMPT. const historyItems = payload.input.filter(item => item.type !== 'compaction_trigger'); @@ -332,7 +286,6 @@ const simulateCompaction = async (ctx: OpenAIResponsesInvocation, gatewayCtx: Ch // conversation history. store: false, }; -<<<<<<<< HEAD:packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts }; // A summarization that closed no assistant text produced no summary, and a @@ -342,72 +295,3 @@ export const EMPTY_SUMMARY_MESSAGE = 'Responses compact shim: the summarization export const containsCompactionTrigger = (input: readonly ResponsesInputItem[]): boolean => input.some(item => item.type === 'compaction_trigger'); -======== - // Pivot the action so the inner dispatch routes to the upstream's - // generate wire instead of its compact wire. The mutation is one-way: - // the project's interceptor convention is that every `ctx.*` write - // propagates downstream and is never restored on the way out. Post-chain - // consumers keep their own captured copies of inputs they care about. - ctx.action = 'generate'; - - const upstreamResult = await run(); - - if (upstreamResult.type !== 'events') { - // api-error / internal-error from the upstream propagate so the client - // learns the compaction failed rather than receiving a silent empty - // envelope. - return upstreamResult; - } - - const closedItems = new Map(); - const observed = (async function* (): AsyncIterable> { - for await (const frame of upstreamResult.events) { - if (frame.type === 'event' && frame.event.type === 'response.output_item.done') { - closedItems.set(frame.event.output_index, frame.event.item); - } - yield frame; - } - })(); - - const collected = await collectOpenAIResponsesProtocolEventsToResult(observed); - const summaryText = summaryTextFrom(closedItems, collected.output); - // A compaction blob is the whole of what the next turn inherits, so an empty - // one silently discards the conversation. - if (summaryText.length === 0) { - throw new Error('OpenAI Responses compact shim: the summarization turn closed no assistant text to summarize'); - } - const cmpId = createRandomOpenAIResponsesItemId('compaction'); - const synthesized = buildCompactionEnvelope(cmpId, summaryText, collected); - - // A real generate turn backs this envelope, so its own `incomplete` or - // `failed` status must reach the terminal event. - return { - ...upstreamResult, - events: syntheticEventsFromResult(synthesized), - }; -}; - -export const containsCompactionTrigger = (input: readonly OpenAIResponsesInputItem[]): boolean => - input.some(item => item.type === 'compaction_trigger'); - -export const withOpenAIResponsesCompactShim: OpenAIResponsesInterceptor = async (ctx, gatewayCtx, run) => { - // The shim is engaged when the operator turned it on for this upstream, - // OR when the upstream's targetApi is not OpenAI Responses (Anthropic Messages / - // OpenAI Chat Completions have no compaction wire and would crash on the - // unknown `compaction_trigger` input variant). - const flagOn = providerModelOf(ctx.candidate).enabledFlags.has('responses-compact-shim'); - const structurallyRequired = ctx.targetApi !== 'openaiResponses'; - if (!flagOn && !structurallyRequired) return await run(); - - // Inbound: expand any prior shim-encoded compactions back into their - // original items so the upstream sees the summarized history. - ctx.payload = expandShimCompactionItems(ctx.payload); - - // Compact-shaped requests are either the native `/responses/compact` - // action or a `generate` call whose input ends in a `compaction_trigger`. - const isCompactShaped = ctx.action === 'compact' || containsCompactionTrigger(ctx.payload.input); - if (!isCompactShaped) return await run(); - - return await simulateCompaction(ctx, gatewayCtx, run); -}; ->>>>>>>> feat/pipeline-non-chat:packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts From 0d41a6cf148e2e2434fc4241e1aac11e9359a584 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 05:40:28 +0800 Subject: [PATCH 078/156] refactor(chat): spell the four chat protocol names in full in the pipelines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rename could not reach the chat chains: they were written on this branch, after the rename PR was cut against main. This is the mechanical half — imports, identifiers and fact keys. chatCompletions -> openaiChatCompletions / OpenAIChatCompletions responses -> openaiResponses / OpenAIResponses messages -> anthropicMessages / AnthropicMessages gemini -> geminiGenerateContent / GeminiGenerateContent Fact keys keep their dotted layout and move only the protocol segment, so `request.chat.chatCompletions` becomes `request.chat.openaiChatCompletions` and `response.chat.messages.streamedUsage` becomes `response.chat.anthropicMessages.streamedUsage`. --- .../data-plane/chat-interceptors_test.ts | 618 ++++++++--------- .../chat/anthropic-messages/errors_test.ts | 2 +- .../chat/anthropic-messages/http_test.ts | 26 +- .../chat/chatcompletions-pipeline_test.ts | 80 +-- .../chat/gemini-count-tokens_test.ts | 42 +- .../data-plane/chat/gemini-pipeline_test.ts | 78 +-- .../chat/messages-count-tokens_test.ts | 50 +- .../data-plane/chat/messages-pipeline_test.ts | 50 +- .../openai-responses/compact-shim_test.ts | 12 +- .../chat/openai-responses/errors_test.ts | 6 +- .../chat/openai-responses/http_test.ts | 2 +- .../chat/openai-responses/websocket_test.ts | 38 +- .../data-plane/chat/responses-compact_test.ts | 40 +- .../chat/responses-pipeline_test.ts | 80 +-- .../data-plane/chat/routes-gemini_test.ts | 24 +- .../data-plane/chat/routes-messages_test.ts | 8 +- .../data-plane/chat/run-dump-frames_test.ts | 12 +- .../data-plane/chat/translated-wires_test.ts | 182 ++--- .../data-plane/pipeline/assembly_test.ts | 28 +- .../chat/anthropic-messages/count-tokens.ts | 98 +-- .../chat/anthropic-messages/errors.ts | 2 +- .../chat/anthropic-messages/http.ts | 34 +- .../interceptors/answer-claude-code-probe.ts | 14 +- .../anthropic-messages/interceptors/types.ts | 16 +- .../chat/anthropic-messages/pipeline.ts | 174 ++--- packages/gateway/src/data-plane/chat/facts.ts | 34 +- .../gemini-generate-content/count-tokens.ts | 92 +-- .../chat/gemini-generate-content/errors.ts | 4 +- .../chat/gemini-generate-content/http.ts | 38 +- .../chat/gemini-generate-content/pipeline.ts | 150 ++-- .../src/data-plane/chat/interceptors.ts | 650 +++++++++--------- .../chat/openai-chat-completions/http.ts | 26 +- .../chat/openai-chat-completions/pipeline.ts | 152 ++-- .../chat/openai-responses/client-output.ts | 12 +- .../chat/openai-responses/compact-shim.ts | 8 +- .../chat/openai-responses/compact.ts | 108 +-- .../chat/openai-responses/errors.ts | 2 +- .../data-plane/chat/openai-responses/http.ts | 28 +- .../openai-responses/interceptors/types.ts | 12 +- .../chat/openai-responses/items/store.ts | 64 +- .../chat/openai-responses/pipeline.ts | 264 +++---- .../chat/openai-responses/serve-prep.ts | 2 +- .../chat/openai-responses/websocket.ts | 72 +- .../gateway/src/data-plane/chat/prologue.ts | 2 +- 44 files changed, 1718 insertions(+), 1718 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index f95745b5e0..72b00a4a7d 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -33,11 +33,11 @@ import { } from '../../src/data-plane/chat/interceptors.ts'; import type { AttemptSelector } from '../../src/data-plane/pipeline/facts.ts'; import { compose, defineStage, move, run } from '@floway-dev/pipeline'; -import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiPayload, GeminiStreamEvent } from '@floway-dev/protocols/gemini'; -import type { MessagesPayload } from '@floway-dev/protocols/messages'; -import type { ResponsesInputItem, ResponsesPayload, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; /** What an attempt is, to a stage that only reads flags. The selector carries them as data, * so nothing here needs the live candidate the resolver kept. */ @@ -109,34 +109,34 @@ const drain = async (answer: unknown): Promise[]> return collected; }; -const chatCompletionsPayload = ( - messages: ChatCompletionsPayload['messages'], - rest: Partial = {}, -): ChatCompletionsPayload => ({ model: 'm', messages, ...rest }) as ChatCompletionsPayload; +const openaiChatCompletionsPayload = ( + messages: OpenAIChatCompletionsPayload['messages'], + rest: Partial = {}, +): OpenAIChatCompletionsPayload => ({ model: 'm', messages, ...rest }) as OpenAIChatCompletionsPayload; -const messagesPayload = (messages: unknown[], rest: Record = {}): MessagesPayload => - ({ model: 'm', max_tokens: 16, messages, ...rest }) as unknown as MessagesPayload; +const anthropicMessagesPayload = (messages: unknown[], rest: Record = {}): AnthropicMessagesPayload => + ({ model: 'm', max_tokens: 16, messages, ...rest }) as unknown as AnthropicMessagesPayload; -const responsesPayload = (input: unknown[], rest: Record = {}): ResponsesPayload => - ({ model: 'm', input, ...rest }) as unknown as ResponsesPayload; +const openaiResponsesPayload = (input: unknown[], rest: Record = {}): OpenAIResponsesPayload => + ({ model: 'm', input, ...rest }) as unknown as OpenAIResponsesPayload; describe('the Chat Completions interceptors, as stages', () => { it('rewrites the roles its flags name, in the settled order', async () => { - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([ + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ { role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }, { role: 'system', content: 'mid' }, ]), 'route.attempt': attemptWith('rewrite-system-to-developer'), }); - const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; + const after = down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload; expect(after.messages.map(message => message.role)).toEqual(['developer', 'user', 'developer']); }); it('turns only a mid-conversation system message into a user one', async () => { - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([ + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ { role: 'system', content: 'lead' }, { role: 'system', content: 'still lead' }, { role: 'user', content: 'hi' }, @@ -144,7 +144,7 @@ describe('the Chat Completions interceptors, as stages', () => { ]), 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), }); - const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; + const after = down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload; expect(after.messages.map(message => message.role)).toEqual(['system', 'system', 'user', 'user']); }); @@ -152,123 +152,123 @@ describe('the Chat Completions interceptors, as stages', () => { // fire must hand the payload back as the *same object*, or every layer costs a copy of // the conversation and the dump stops being affordable. it('hands the same payload back by identity when no flag fires', async () => { - const original = chatCompletionsPayload([{ role: 'system', content: 'lead' }]); - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': original, + const original = openaiChatCompletionsPayload([{ role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith(), }); - expect(down['request.chat.chatCompletions']).toBe(original); + expect(down['request.chat.openaiChatCompletions']).toBe(original); }); it('hands the same payload back when a flag fires but changes nothing', async () => { - const original = chatCompletionsPayload([{ role: 'user', content: 'hi' }]); - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': original, + const original = openaiChatCompletionsPayload([{ role: 'user', content: 'hi' }]); + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('rewrite-system-to-developer'), }); - expect(down['request.chat.chatCompletions']).toBe(original); + expect(down['request.chat.openaiChatCompletions']).toBe(original); }); it('shares every message it did not rewrite', async () => { const messages = Array.from({ length: 49 }, (_, index) => ( index === 0 ? { role: 'system' as const, content: 'lead' } : { role: 'user' as const, content: `turn ${index}` } )); - const original = chatCompletionsPayload(messages); - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': original, + const original = openaiChatCompletionsPayload(messages); + const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('rewrite-system-to-developer'), }); - const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; + const after = down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload; const shared = after.messages.filter((message, index) => message === messages[index]).length; expect(shared).toBe(48); // every message but the one that changed }); it('disables reasoning only when a tool choice is forced', async () => { - const { down: forced } = await runStage(disableReasoningOnForcedToolChoiceForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([], { tool_choice: 'required' }), + const { down: forced } = await runStage(disableReasoningOnForcedToolChoiceForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { tool_choice: 'required' }), 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); - expect((forced['request.chat.chatCompletions'] as ChatCompletionsPayload).reasoning_effort).toBe('none'); + expect((forced['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).reasoning_effort).toBe('none'); - const free = chatCompletionsPayload([], { tool_choice: 'auto' }); - const { down: untouched } = await runStage(disableReasoningOnForcedToolChoiceForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': free, + const free = openaiChatCompletionsPayload([], { tool_choice: 'auto' }); + const { down: untouched } = await runStage(disableReasoningOnForcedToolChoiceForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': free, 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); - expect(untouched['request.chat.chatCompletions']).toBe(free); + expect(untouched['request.chat.openaiChatCompletions']).toBe(free); }); it('removes the prompt cache key, and the key itself rather than its value', async () => { - const { down } = await runStage(stripPromptCacheKeyForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([], { prompt_cache_key: 'k' } as Partial), + const { down } = await runStage(stripPromptCacheKeyForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { prompt_cache_key: 'k' } as Partial), 'route.attempt': attemptWith('strip-prompt-cache-key'), }); - const after = down['request.chat.chatCompletions'] as ChatCompletionsPayload; + const after = down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload; // `undefined` is not a removal: the key has to be gone, not present and empty. expect('prompt_cache_key' in after).toBe(false); }); it('leaves a payload that never had the key alone, by identity', async () => { - const original = chatCompletionsPayload([]); - const { down } = await runStage(stripPromptCacheKeyForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': original, + const original = openaiChatCompletionsPayload([]); + const { down } = await runStage(stripPromptCacheKeyForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('strip-prompt-cache-key'), }); - expect(down['request.chat.chatCompletions']).toBe(original); + expect(down['request.chat.openaiChatCompletions']).toBe(original); }); // The usage chunk is what every stream here is billed from, so what the client asked for and // what the upstream is asked for differ — and this is the rule that makes them differ. it('asks the upstream for the usage chunk when the client sent no stream options', async () => { - const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([]), + const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), }); - expect((down['request.chat.chatCompletions'] as ChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); + expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); }); it('asks for it over a client that asked for it to be left out', async () => { - const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([], { stream_options: { include_usage: false } }), + const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { stream_options: { include_usage: false } }), }); - expect((down['request.chat.chatCompletions'] as ChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); + expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); }); // Everything else on `stream_options` is the client's own. An upstream that reads a key our // typed surface does not name still gets it. it('keeps the client-s other stream options while it forces the one it wants', async () => { - const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([], { - stream_options: { extra: 'keep-me', include_usage: false } as unknown as ChatCompletionsPayload['stream_options'], + const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { + stream_options: { extra: 'keep-me', include_usage: false } as unknown as OpenAIChatCompletionsPayload['stream_options'], }), }); - expect((down['request.chat.chatCompletions'] as ChatCompletionsPayload).stream_options) + expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).stream_options) .toEqual({ extra: 'keep-me', include_usage: true }); }); it('leaves a payload that already asked for it alone, by identity', async () => { - const original = chatCompletionsPayload([], { stream_options: { include_usage: true } }); - const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': original, + const original = openaiChatCompletionsPayload([], { stream_options: { include_usage: true } }); + const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, }); - expect(down['request.chat.chatCompletions']).toBe(original); + expect(down['request.chat.openaiChatCompletions']).toBe(original); }); // The spec puts the final usage on a carrier chunk of its own; an upstream that hangs it off // the last delta is split in two so that everything downstream reads the one shape. it('moves usage off the chunk that carried the last delta and onto a carrier of its own', async () => { - const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.chatCompletions', {}, - streamOf([eventFrame({ + const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.openaiChatCompletions', {}, + streamOf([eventFrame({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 }, - } as unknown as ChatCompletionsStreamEvent)])); - const frames = await drain(up['response.chat.chatCompletions']); + } as unknown as OpenAIChatCompletionsStreamEvent)])); + const frames = await drain(up['response.chat.openaiChatCompletions']); expect(frames).toHaveLength(2); - const delta = chatCompletionsEventOf(frames[0]!); + const delta = openaiChatCompletionsEventOf(frames[0]!); expect(delta.choices).toEqual([{ index: 0, delta: {}, finish_reason: 'stop' }]); expect('usage' in delta).toBe(false); - const carrier = chatCompletionsEventOf(frames[1]!); + const carrier = openaiChatCompletionsEventOf(frames[1]!); // The carrier is the same chunk otherwise: an id and a model a client can correlate. expect(carrier).toMatchObject({ id: 'c1', model: 'm', choices: [] }); expect(carrier.usage).toEqual({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 }); @@ -278,16 +278,16 @@ describe('the Chat Completions interceptors, as stages', () => { const carrier = eventFrame({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_tokens_details: { cached_tokens: 60 } }, - } as unknown as ChatCompletionsStreamEvent); + } as unknown as OpenAIChatCompletionsStreamEvent); const content = eventFrame({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - } as unknown as ChatCompletionsStreamEvent); + } as unknown as OpenAIChatCompletionsStreamEvent); const done = doneFrame(); - const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.chatCompletions', {}, - streamOf([carrier, content, done])); - const frames = await drain(up['response.chat.chatCompletions']); + const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.openaiChatCompletions', {}, + streamOf([carrier, content, done])); + const frames = await drain(up['response.chat.openaiChatCompletions']); expect(frames).toEqual([carrier, content, done]); }); @@ -295,13 +295,13 @@ describe('the Chat Completions interceptors, as stages', () => { // Naming a vendor's cache fields is the vendor stage's job, and it has already run by the // time the answer reaches this one. A chunk still carrying them is one no vendor claimed. it('leaves a vendor-s own cache fields where it found them', async () => { - const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.chatCompletions', {}, - streamOf([chatCompletionsUsageChunk({ + const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.openaiChatCompletions', {}, + streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 25, })])); - const frames = await drain(up['response.chat.chatCompletions']); + const frames = await drain(up['response.chat.openaiChatCompletions']); - expect(chatCompletionsUsageOf(frames[0]!)).toEqual({ + expect(openaiChatCompletionsUsageOf(frames[0]!)).toEqual({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 25, }); }); @@ -310,14 +310,14 @@ describe('the Chat Completions interceptors, as stages', () => { // counts only what the cache did not serve, and `total_tokens` witnesses it whatever anyone // flagged. it('folds the cache buckets into the input total when the totals witness it', async () => { - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith(), - }, streamOf([chatCompletionsUsageChunk({ + }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_tokens_details: { cached_tokens: 13312 }, })])); - const frames = await drain(up['response.chat.chatCompletions']); + const frames = await drain(up['response.chat.openaiChatCompletions']); - expect(chatCompletionsUsageOf(frames[0]!)).toMatchObject({ + expect(openaiChatCompletionsUsageOf(frames[0]!)).toMatchObject({ prompt_tokens: 13791, // The total already counted the real input, and now agrees with it. total_tokens: 14164, @@ -329,114 +329,114 @@ describe('the Chat Completions interceptors, as stages', () => { // input rather than the gate — and it arrives on the attempt this stage read on the way // down, so this is also what proves the closure carries it to the way back. it('folds on the flag alone when the chunk states no total, cache writes included', async () => { - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), - }, streamOf([chatCompletionsUsageChunk({ + }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 10, prompt_tokens_details: { cached_tokens: 120, cache_creation_input_tokens: 80 }, })])); - const frames = await drain(up['response.chat.chatCompletions']); + const frames = await drain(up['response.chat.openaiChatCompletions']); - expect(chatCompletionsUsageOf(frames[0]!)).toMatchObject({ prompt_tokens: 300 }); + expect(openaiChatCompletionsUsageOf(frames[0]!)).toMatchObject({ prompt_tokens: 300 }); }); it('hands an inclusive chunk, and one with no cache counts, back by identity', async () => { - const inclusive = chatCompletionsUsageChunk({ + const inclusive = openaiChatCompletionsUsageChunk({ prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_tokens_details: { cached_tokens: 400 }, }); - const uncached = chatCompletionsUsageChunk({ prompt_tokens: 40, completion_tokens: 1, total_tokens: 41 }); + const uncached = openaiChatCompletionsUsageChunk({ prompt_tokens: 40, completion_tokens: 1, total_tokens: 41 }); - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), - }, streamOf([uncached])); - expect(await drain(up['response.chat.chatCompletions'])).toEqual([uncached]); + }, streamOf([uncached])); + expect(await drain(up['response.chat.openaiChatCompletions'])).toEqual([uncached]); - const { up: unflagged } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + const { up: unflagged } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith(), - }, streamOf([inclusive])); - expect(await drain(unflagged['response.chat.chatCompletions'])).toEqual([inclusive]); + }, streamOf([inclusive])); + expect(await drain(unflagged['response.chat.openaiChatCompletions'])).toEqual([inclusive]); }); // Both contradictions name the flag and the upstream, because the remedy is a setting an // operator makes on that upstream. Neither may pass silently: one over-charges the input by // the whole cached prefix, the other underflows it. it('raises when the flag claims exclusive and the totals say inclusive', async () => { - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), - }, streamOf([chatCompletionsUsageChunk({ + }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_tokens_details: { cached_tokens: 400 }, })])); - await expect(drain(up['response.chat.chatCompletions'])) + await expect(drain(up['response.chat.openaiChatCompletions'])) .rejects.toThrow(/usage-exclusive-cached-tokens is enabled/); }); it('raises naming the flag when the cache counts underflow with no verdict', async () => { - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith(), - }, streamOf([chatCompletionsUsageChunk({ + }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 479, completion_tokens: 373, prompt_tokens_details: { cached_tokens: 13312 }, })])); - await expect(drain(up['response.chat.chatCompletions'])) + await expect(drain(up['response.chat.openaiChatCompletions'])) .rejects.toThrow(/enable usage-exclusive-cached-tokens/); }); it('leaves an answer that is not a stream alone, by identity', async () => { const answer = { kind: 'value' as const, body: null }; - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.chatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith(), }, answer); - expect(up['response.chat.chatCompletions']).toBe(answer); + expect(up['response.chat.openaiChatCompletions']).toBe(answer); }); }); /** A `choices: []` carrier holding nothing but the counts under test — the shape the spec puts * a stream's final usage on. */ -const chatCompletionsUsageChunk = (usage: Record): ProtocolFrame => +const openaiChatCompletionsUsageChunk = (usage: Record): ProtocolFrame => eventFrame({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage, - } as unknown as ChatCompletionsStreamEvent); + } as unknown as OpenAIChatCompletionsStreamEvent); -const chatCompletionsEventOf = (frame: ProtocolFrame): Record => +const openaiChatCompletionsEventOf = (frame: ProtocolFrame): Record => (frame as unknown as { event: Record }).event; -const chatCompletionsUsageOf = (frame: ProtocolFrame): Record => - chatCompletionsEventOf(frame).usage as Record; +const openaiChatCompletionsUsageOf = (frame: ProtocolFrame): Record => + openaiChatCompletionsEventOf(frame).usage as Record; -const deepSeekDelta = (delta: Record): ProtocolFrame => +const deepSeekDelta = (delta: Record): ProtocolFrame => eventFrame({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta, finish_reason: null }], - } as unknown as ChatCompletionsStreamEvent); + } as unknown as OpenAIChatCompletionsStreamEvent); -const deltaOf = (frame: ProtocolFrame): Record => - (chatCompletionsEventOf(frame).choices as { delta: Record }[])[0]!.delta; +const deltaOf = (frame: ProtocolFrame): Record => + (openaiChatCompletionsEventOf(frame).choices as { delta: Record }[])[0]!.delta; /** An assistant turn as the gateway holds it: the canonical reasoning fields, and a tool call * the replay of a multi-turn loop has to keep. */ -const assistantReplay = (reasoning: Record): ChatCompletionsPayload['messages'] => [ +const assistantReplay = (reasoning: Record): OpenAIChatCompletionsPayload['messages'] => [ { role: 'user', content: 'first turn' }, { role: 'assistant', content: null, ...reasoning, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{}' } }], }, { role: 'tool', tool_call_id: 'call_1', content: 'result' }, -] as unknown as ChatCompletionsPayload['messages']; +] as unknown as OpenAIChatCompletionsPayload['messages']; const assistantOf = (payload: unknown): Record => - (payload as ChatCompletionsPayload).messages[1] as unknown as Record; + (payload as OpenAIChatCompletionsPayload).messages[1] as unknown as Record; describe('the Chat Completions vendor dialects, as stages', () => { it('projects the canonical assistant reasoning onto the one field DeepSeek reads', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload(assistantReplay({ + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload(assistantReplay({ reasoning_text: 'let me check the docs', reasoning_opaque: 'opaque-blob', reasoning_items: [{ type: 'reasoning', summary: [] }], })), 'route.attempt': attemptWith('vendor-deepseek'), }); - const assistant = assistantOf(down['request.chat.chatCompletions']); + const assistant = assistantOf(down['request.chat.openaiChatCompletions']); expect(assistant.reasoning_content).toBe('let me check the docs'); // The other three are not DeepSeek's, and `reasoning_opaque` in particular is the @@ -448,8 +448,8 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('writes that field from the summary items when the scalar is the thing that is missing', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload(assistantReplay({ + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload(assistantReplay({ reasoning_items: [{ type: 'reasoning', id: 'rs_1', summary: [{ type: 'summary_text', text: 'step one. ' }, { type: 'summary_text', text: 'step two.' }], @@ -458,18 +458,18 @@ describe('the Chat Completions vendor dialects, as stages', () => { 'route.attempt': attemptWith('vendor-deepseek'), }); - expect(assistantOf(down['request.chat.chatCompletions']).reasoning_content).toBe('step one. step two.'); + expect(assistantOf(down['request.chat.openaiChatCompletions']).reasoning_content).toBe('step one. step two.'); }); it('strips the fields it cannot project even when there is nothing to project', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([ + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ { role: 'user', content: 'first turn' }, { role: 'assistant', content: 'answer', reasoning_items: [{ type: 'reasoning' }], reasoning_opaque: 'opaque-chain' }, - ] as unknown as ChatCompletionsPayload['messages']), + ] as unknown as OpenAIChatCompletionsPayload['messages']), 'route.attempt': attemptWith('vendor-deepseek'), }); - const assistant = assistantOf(down['request.chat.chatCompletions']); + const assistant = assistantOf(down['request.chat.openaiChatCompletions']); expect('reasoning_content' in assistant).toBe(false); expect('reasoning_items' in assistant).toBe(false); @@ -478,22 +478,22 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('puts the reasoning sentinel into the shape DeepSeek reads', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([], { reasoning_effort: 'none' }), + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { reasoning_effort: 'none' }), 'route.attempt': attemptWith('vendor-deepseek'), }); - const after = down['request.chat.chatCompletions'] as Record; + const after = down['request.chat.openaiChatCompletions'] as Record; expect('reasoning_effort' in after).toBe(false); expect(after.thinking).toEqual({ type: 'disabled' }); }); it('puts the reasoning sentinel into the shape Qwen reads', async () => { - const { down } = await runStage(vendorQwenNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([], { reasoning_effort: 'none' }), + const { down } = await runStage(vendorQwenNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { reasoning_effort: 'none' }), 'route.attempt': attemptWith('vendor-qwen'), }); - const after = down['request.chat.chatCompletions'] as Record; + const after = down['request.chat.openaiChatCompletions'] as Record; expect('reasoning_effort' in after).toBe(false); expect(after.enable_thinking).toBe(false); @@ -502,21 +502,21 @@ describe('the Chat Completions vendor dialects, as stages', () => { // Only the sentinel means "no reasoning". A real effort is the client asking for reasoning, // and DeepSeek and Qwen both take it in the canonical field. it('leaves a payload that asked for real reasoning alone, by identity', async () => { - const original = chatCompletionsPayload([], { reasoning_effort: 'high' }); + const original = openaiChatCompletionsPayload([], { reasoning_effort: 'high' }); for (const stage of [vendorDeepSeekNormalizeForChatCompletions, vendorQwenNormalizeForChatCompletions]) { - const { down } = await runStage(stage, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': original, + const { down } = await runStage(stage, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('vendor-deepseek', 'vendor-qwen'), }); - expect(down['request.chat.chatCompletions']).toBe(original); + expect(down['request.chat.openaiChatCompletions']).toBe(original); } }); // DeepSeek's structured output takes `json_object` and nothing else, so the schema goes // rather than the request being refused for carrying it. it('downgrades a JSON schema to the JSON object DeepSeek supports', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([], { + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { response_format: { type: 'json_schema', json_schema: { name: 'messages_response', strict: true, schema: { type: 'object' } }, @@ -525,29 +525,29 @@ describe('the Chat Completions vendor dialects, as stages', () => { 'route.attempt': attemptWith('vendor-deepseek'), }); - expect((down['request.chat.chatCompletions'] as ChatCompletionsPayload).response_format).toEqual({ type: 'json_object' }); + expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).response_format).toEqual({ type: 'json_object' }); }); it('leaves a payload already asking for a JSON object alone, by identity', async () => { - const original = chatCompletionsPayload([], { response_format: { type: 'json_object' } }); - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': original, + const original = openaiChatCompletionsPayload([], { response_format: { type: 'json_object' } }); + const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('vendor-deepseek'), }); - expect(down['request.chat.chatCompletions']).toBe(original); + expect(down['request.chat.openaiChatCompletions']).toBe(original); }); it('renames DeepSeek-s reasoning deltas to the canonical field', async () => { - const { up } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([]), + const { up } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), 'route.attempt': attemptWith('vendor-deepseek'), - }, streamOf([ + }, streamOf([ deepSeekDelta({ role: 'assistant' }), deepSeekDelta({ reasoning_content: 'thinking...' }), deepSeekDelta({ content: 'answer' }), doneFrame(), ])); - const frames = await drain(up['response.chat.chatCompletions']); + const frames = await drain(up['response.chat.openaiChatCompletions']); expect(deltaOf(frames[1]!).reasoning_text).toBe('thinking...'); expect('reasoning_content' in deltaOf(frames[1]!)).toBe(false); @@ -557,16 +557,16 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('rewrites DeepSeek-s cache-hit count into the field the rules above it read', async () => { - const { up } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([]), + const { up } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), 'route.attempt': attemptWith('vendor-deepseek'), - }, streamOf([chatCompletionsUsageChunk({ + }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, prompt_cache_miss_tokens: 30, prompt_tokens_details: { upstream_metric: 'preserved' }, })])); - const frames = await drain(up['response.chat.chatCompletions']); - const usage = chatCompletionsUsageOf(frames[0]!); + const frames = await drain(up['response.chat.openaiChatCompletions']); + const usage = openaiChatCompletionsUsageOf(frames[0]!); expect(usage.prompt_tokens).toBe(100); expect(usage.prompt_tokens_details).toEqual({ upstream_metric: 'preserved', cached_tokens: 70 }); @@ -576,14 +576,14 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('rewrites Kimi-s flat cached count into that same field', async () => { - const { up } = await runStage(vendorKimiNormalizeForChatCompletions, 'response.chat.chatCompletions', { + const { up } = await runStage(vendorKimiNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith('vendor-kimi'), - }, streamOf([chatCompletionsUsageChunk({ + }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, cached_tokens: 50, prompt_tokens_details: { upstream_metric: 'preserved' }, })])); - const frames = await drain(up['response.chat.chatCompletions']); - const usage = chatCompletionsUsageOf(frames[0]!); + const frames = await drain(up['response.chat.openaiChatCompletions']); + const usage = openaiChatCompletionsUsageOf(frames[0]!); expect(usage.prompt_tokens_details).toEqual({ upstream_metric: 'preserved', cached_tokens: 50 }); expect('cached_tokens' in usage).toBe(false); @@ -597,24 +597,24 @@ describe('the Chat Completions vendor dialects, as stages', () => { [vendorDeepSeekNormalizeForChatCompletions, 'vendor-deepseek', 'prompt_cache_hit_tokens'], [vendorKimiNormalizeForChatCompletions, 'vendor-kimi', 'cached_tokens'], ] as const) { - const { up } = await runStage(stage, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([]), + const { up } = await runStage(stage, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), 'route.attempt': attemptWith(flag), - }, streamOf([chatCompletionsUsageChunk({ + }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, [vendorField]: 70, prompt_tokens_details: [{ cached_tokens: 1 }], })])); - const frames = await drain(up['response.chat.chatCompletions']); + const frames = await drain(up['response.chat.openaiChatCompletions']); - expect(chatCompletionsUsageOf(frames[0]!).prompt_tokens_details).toEqual({ cached_tokens: 70 }); + expect(openaiChatCompletionsUsageOf(frames[0]!).prompt_tokens_details).toEqual({ cached_tokens: 70 }); } }); // A flag is data on an upstream model. A candidate that does not carry one is an upstream // whose wire is already canonical, and nothing here may touch it. it('stands down on a candidate that carries none of the vendor flags', async () => { - const payload = chatCompletionsPayload([], { reasoning_effort: 'none' }); - const usage = chatCompletionsUsageChunk({ + const payload = openaiChatCompletionsPayload([], { reasoning_effort: 'none' }); + const usage = openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 50, }); for (const stage of [ @@ -622,13 +622,13 @@ describe('the Chat Completions vendor dialects, as stages', () => { vendorQwenNormalizeForChatCompletions, vendorKimiNormalizeForChatCompletions, ]) { - const { down, up } = await runStage(stage, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': payload, + const { down, up } = await runStage(stage, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': payload, 'route.attempt': attemptWith(), - }, streamOf([usage])); + }, streamOf([usage])); - expect(down['request.chat.chatCompletions']).toBe(payload); - expect(await drain(up['response.chat.chatCompletions'])).toEqual([usage]); + expect(down['request.chat.openaiChatCompletions']).toBe(payload); + expect(await drain(up['response.chat.openaiChatCompletions'])).toEqual([usage]); } }); @@ -640,59 +640,59 @@ describe('the Chat Completions vendor dialects, as stages', () => { [vendorDeepSeekNormalizeForChatCompletions, 'vendor-deepseek'], [vendorKimiNormalizeForChatCompletions, 'vendor-kimi'], ] as const) { - const { up } = await runStage(stage, 'response.chat.chatCompletions', { - 'request.chat.chatCompletions': chatCompletionsPayload([]), + const { up } = await runStage(stage, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), 'route.attempt': attemptWith(flag), }, { kind: 'stream' as const, frames: { [Symbol.asyncIterator]: () => (async function* () { throw failure; })() } }); - await expect(drain(up['response.chat.chatCompletions'])).rejects.toBe(failure); + await expect(drain(up['response.chat.openaiChatCompletions'])).rejects.toBe(failure); } }); }); describe('the Messages interceptors, as stages', () => { it('turns every inline system message into a user one', async () => { - const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.messages', { - 'request.chat.messages': messagesPayload([ + const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([ { role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }, { role: 'system', content: 'mid' }, ]), 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), }); - const after = down['request.chat.messages'] as MessagesPayload; + const after = down['request.chat.anthropicMessages'] as AnthropicMessagesPayload; // The top-level `system` field is the only first-position slot on this protocol, so even // the leading one is past the leading run. expect(after.messages.map(message => message.role)).toEqual(['user', 'user', 'user']); }); it('hands the same conversation back by identity when the flag does not fire', async () => { - const original = messagesPayload([{ role: 'system', content: 'lead' }]); - const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.messages', { - 'request.chat.messages': original, + const original = anthropicMessagesPayload([{ role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': original, 'route.attempt': attemptWith(), }); - expect(down['request.chat.messages']).toBe(original); + expect(down['request.chat.anthropicMessages']).toBe(original); }); it('hands the same conversation back when the flag fires but there is no system message', async () => { - const original = messagesPayload([{ role: 'user', content: 'hi' }]); - const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.messages', { - 'request.chat.messages': original, + const original = anthropicMessagesPayload([{ role: 'user', content: 'hi' }]); + const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': original, 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), }); - expect(down['request.chat.messages']).toBe(original); + expect(down['request.chat.anthropicMessages']).toBe(original); }); it('disables thinking on a forced tool choice and leaves the structured-output format', async () => { - const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.messages', { - 'request.chat.messages': messagesPayload([], { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([], { tool_choice: { type: 'any' }, output_config: { effort: 'high', format: { type: 'json_schema', schema: {} } }, }), 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); - const after = down['request.chat.messages'] as MessagesPayload; + const after = down['request.chat.anthropicMessages'] as AnthropicMessagesPayload; expect(after.thinking).toEqual({ type: 'disabled' }); // Forced tool choice composes fine with structured output; it is thinking it does not // compose with, so only the effort goes. @@ -700,61 +700,61 @@ describe('the Messages interceptors, as stages', () => { }); it('drops an output_config that held nothing but the effort', async () => { - const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.messages', { - 'request.chat.messages': messagesPayload([], { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([], { tool_choice: { type: 'tool', name: 'f' }, output_config: { effort: 'high' }, }), 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); - expect('output_config' in (down['request.chat.messages'] as MessagesPayload)).toBe(false); + expect('output_config' in (down['request.chat.anthropicMessages'] as AnthropicMessagesPayload)).toBe(false); }); it('leaves a free tool choice alone, by identity', async () => { - const free = messagesPayload([], { tool_choice: { type: 'auto' } }); - const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.messages', { - 'request.chat.messages': free, + const free = anthropicMessagesPayload([], { tool_choice: { type: 'auto' } }); + const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': free, 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); - expect(down['request.chat.messages']).toBe(free); + expect(down['request.chat.anthropicMessages']).toBe(free); }); it('scrubs the billing header line and the per-turn hash out of the system prompt', async () => { - const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.messages', { - 'request.chat.messages': messagesPayload([], { + const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([], { system: 'x-anthropic-billing-header: cch=abcdef12\nreal instructions\ncch=beef01;', }), 'route.attempt': attemptWith('strip-billing-attribution'), }); - expect((down['request.chat.messages'] as MessagesPayload).system).toBe('real instructions'); + expect((down['request.chat.anthropicMessages'] as AnthropicMessagesPayload).system).toBe('real instructions'); }); it('drops a system block the scrub emptied, and the field when nothing is left', async () => { - const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.messages', { - 'request.chat.messages': messagesPayload([], { + const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([], { system: [{ type: 'text', text: 'x-anthropic-billing-header: cch=abcdef12' }], }), 'route.attempt': attemptWith('strip-billing-attribution'), }); // An empty system prompt is not a system prompt: the field goes rather than riding on // empty. - expect('system' in (down['request.chat.messages'] as MessagesPayload)).toBe(false); + expect('system' in (down['request.chat.anthropicMessages'] as AnthropicMessagesPayload)).toBe(false); }); it('leaves a system prompt carrying no attribution block alone, by identity', async () => { - const original = messagesPayload([], { system: [{ type: 'text', text: 'real instructions' }] }); - const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.messages', { - 'request.chat.messages': original, + const original = anthropicMessagesPayload([], { system: [{ type: 'text', text: 'real instructions' }] }); + const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': original, 'route.attempt': attemptWith('strip-billing-attribution'), }); - expect(down['request.chat.messages']).toBe(original); + expect(down['request.chat.anthropicMessages']).toBe(original); }); }); describe('the Gemini interceptors, as stages', () => { it('drops the part fields no translation carries, and a part they were all of', async () => { - const { down } = await runStage(stripUnsupportedPartFieldsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': { + const { down } = await runStage(stripUnsupportedPartFieldsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { contents: [{ role: 'user', parts: [ @@ -763,61 +763,61 @@ describe('the Gemini interceptors, as stages', () => { ], }], systemInstruction: { parts: [{ text: 'sys', codeExecutionResult: {} }] }, - } satisfies GeminiPayload, + } satisfies GeminiGenerateContentPayload, }); - const after = down['request.chat.gemini'] as GeminiPayload; + const after = down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload; expect(after.contents?.[0]?.parts).toEqual([{ text: 'hi' }]); expect(after.systemInstruction?.parts).toEqual([{ text: 'sys' }]); }); it('hands a payload carrying no unsupported part field back by identity', async () => { - const original: GeminiPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; - const { down } = await runStage(stripUnsupportedPartFieldsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': original, + const original: GeminiGenerateContentPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; + const { down } = await runStage(stripUnsupportedPartFieldsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': original, }); - expect(down['request.chat.gemini']).toBe(original); + expect(down['request.chat.geminiGenerateContent']).toBe(original); }); it('keeps a tool group only for its function declarations', async () => { - const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': { + const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { tools: [{ functionDeclarations: [{ name: 'f' }], googleSearch: {} }, { codeExecution: {} }], - } satisfies GeminiPayload, + } satisfies GeminiGenerateContentPayload, }); - expect((down['request.chat.gemini'] as GeminiPayload).tools).toEqual([{ functionDeclarations: [{ name: 'f' }] }]); + expect((down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload).tools).toEqual([{ functionDeclarations: [{ name: 'f' }] }]); }); it('removes the tool list itself when no group declared a function', async () => { - const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': { tools: [{ googleSearch: {} }] } satisfies GeminiPayload, + const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { tools: [{ googleSearch: {} }] } satisfies GeminiGenerateContentPayload, }); // An empty tool list is a different request from no tools at all. - expect('tools' in (down['request.chat.gemini'] as GeminiPayload)).toBe(false); + expect('tools' in (down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload)).toBe(false); }); it('hands a payload whose groups only declare functions back by identity', async () => { - const original: GeminiPayload = { tools: [{ functionDeclarations: [{ name: 'f' }] }] }; - const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': original, + const original: GeminiGenerateContentPayload = { tools: [{ functionDeclarations: [{ name: 'f' }] }] }; + const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': original, }); - expect(down['request.chat.gemini']).toBe(original); + expect(down['request.chat.geminiGenerateContent']).toBe(original); }); it('removes the safety settings, and leaves a payload that carries none alone', async () => { - const { down } = await runStage(stripSafetySettingsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': { safetySettings: [{ category: 'c', threshold: 't' }] } satisfies GeminiPayload, + const { down } = await runStage(stripSafetySettingsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { safetySettings: [{ category: 'c', threshold: 't' }] } satisfies GeminiGenerateContentPayload, }); - expect('safetySettings' in (down['request.chat.gemini'] as GeminiPayload)).toBe(false); + expect('safetySettings' in (down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload)).toBe(false); - const original: GeminiPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; - const { down: untouched } = await runStage(stripSafetySettingsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': original, + const original: GeminiGenerateContentPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; + const { down: untouched } = await runStage(stripSafetySettingsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': original, }); - expect(untouched['request.chat.gemini']).toBe(original); + expect(untouched['request.chat.geminiGenerateContent']).toBe(original); }); it('hides thought parts from a caller who did not opt in', async () => { - const answer = streamOf([ + const answer = streamOf([ eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }] } }] }), eventFrame({ candidates: [{ @@ -827,46 +827,46 @@ describe('the Gemini interceptors, as stages', () => { }], }), ]); - const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': {} satisfies GeminiPayload, + const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': {} satisfies GeminiGenerateContentPayload, }, answer); - const frames = await drain(up['response.chat.gemini']); + const frames = await drain(up['response.chat.geminiGenerateContent']); // The first event was pure thought and had not finished, so it is nothing at all and // never reaches the client; the second keeps only what the caller asked for. expect(frames).toHaveLength(1); - const event = (frames[0] as { event: Extract }).event; + const event = (frames[0] as { event: Extract }).event; expect(event.candidates?.[0]?.content.parts).toEqual([{ text: 'the answer' }]); }); // The opt-in is something the client sent, so it is read on the way down and used on the // way back — the closure carrying it across is the only mechanism this proves. it('leaves the stream alone when the caller asked to see thoughts', async () => { - const answer = streamOf([ + const answer = streamOf([ eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }] }, finishReason: 'STOP' }] }), ]); - const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': { generationConfig: { thinkingConfig: { includeThoughts: true } } } satisfies GeminiPayload, + const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { generationConfig: { thinkingConfig: { includeThoughts: true } } } satisfies GeminiGenerateContentPayload, }, answer); - expect(up['response.chat.gemini']).toBe(answer); + expect(up['response.chat.geminiGenerateContent']).toBe(answer); }); it('leaves an answer that is not a stream alone, by identity', async () => { const answer = { kind: 'value' as const, body: null }; - const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.gemini', { - 'request.chat.gemini': {} satisfies GeminiPayload, + const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': {} satisfies GeminiGenerateContentPayload, }, answer); - expect(up['response.chat.gemini']).toBe(answer); + expect(up['response.chat.geminiGenerateContent']).toBe(answer); }); }); describe('the Responses interceptors, as stages', () => { const rolesOf = (down: Record): unknown[] => - ((down['request.chat.responses'] as ResponsesPayload).input as ResponsesInputItem[]) + ((down['request.chat.openaiResponses'] as OpenAIResponsesPayload).input as OpenAIResponsesInputItem[]) .map(item => (item as { role?: unknown }).role); it('rewrites the roles its flags name, in the settled order', async () => { - const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.responses', { - 'request.chat.responses': responsesPayload([ + const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([ { type: 'message', role: 'system', content: 'lead' }, { type: 'message', role: 'user', content: 'hi' }, { type: 'message', role: 'system', content: 'mid' }, @@ -879,8 +879,8 @@ describe('the Responses interceptors, as stages', () => { // The ordering the rule depends on: an item with no role of its own still ends the leading // system run, which is what makes the system message after it mid-conversation. it('lets an item carrying no role cross the leading system run', async () => { - const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.responses', { - 'request.chat.responses': responsesPayload([ + const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([ { type: 'message', role: 'system', content: 'lead' }, { type: 'reasoning', id: 'rs_1', summary: [] }, { type: 'message', role: 'system', content: 'after' }, @@ -891,58 +891,58 @@ describe('the Responses interceptors, as stages', () => { }); it('hands the same payload back by identity when no flag fires', async () => { - const original = responsesPayload([{ type: 'message', role: 'system', content: 'lead' }]); - const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.responses', { - 'request.chat.responses': original, + const original = openaiResponsesPayload([{ type: 'message', role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': original, 'route.attempt': attemptWith(), }); - expect(down['request.chat.responses']).toBe(original); + expect(down['request.chat.openaiResponses']).toBe(original); }); it('disables reasoning on a required tool choice and on a named one', async () => { for (const toolChoice of ['required', { type: 'function', name: 'f' }]) { - const { down } = await runStage(disableReasoningOnForcedToolChoiceForResponses, 'response.chat.responses', { - 'request.chat.responses': responsesPayload([], { tool_choice: toolChoice, reasoning: { effort: 'high', summary: 'auto' } }), + const { down } = await runStage(disableReasoningOnForcedToolChoiceForResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([], { tool_choice: toolChoice, reasoning: { effort: 'high', summary: 'auto' } }), 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); // The object is replaced rather than merged: a summary has nothing to summarize once // reasoning is off. - expect((down['request.chat.responses'] as ResponsesPayload).reasoning).toEqual({ effort: 'none' }); + expect((down['request.chat.openaiResponses'] as OpenAIResponsesPayload).reasoning).toEqual({ effort: 'none' }); } }); it('leaves an automatic tool choice alone, by identity', async () => { - const free = responsesPayload([], { tool_choice: 'auto' }); - const { down } = await runStage(disableReasoningOnForcedToolChoiceForResponses, 'response.chat.responses', { - 'request.chat.responses': free, + const free = openaiResponsesPayload([], { tool_choice: 'auto' }); + const { down } = await runStage(disableReasoningOnForcedToolChoiceForResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': free, 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); - expect(down['request.chat.responses']).toBe(free); + expect(down['request.chat.openaiResponses']).toBe(free); }); it('removes the prompt cache key, and leaves a payload that never had it alone', async () => { - const { down } = await runStage(stripPromptCacheKeyForResponses, 'response.chat.responses', { - 'request.chat.responses': responsesPayload([], { prompt_cache_key: 'k' }), + const { down } = await runStage(stripPromptCacheKeyForResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([], { prompt_cache_key: 'k' }), 'route.attempt': attemptWith('strip-prompt-cache-key'), }); - expect('prompt_cache_key' in (down['request.chat.responses'] as ResponsesPayload)).toBe(false); + expect('prompt_cache_key' in (down['request.chat.openaiResponses'] as OpenAIResponsesPayload)).toBe(false); - const original = responsesPayload([]); - const { down: untouched } = await runStage(stripPromptCacheKeyForResponses, 'response.chat.responses', { - 'request.chat.responses': original, + const original = openaiResponsesPayload([]); + const { down: untouched } = await runStage(stripPromptCacheKeyForResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': original, 'route.attempt': attemptWith('strip-prompt-cache-key'), }); - expect(untouched['request.chat.responses']).toBe(original); + expect(untouched['request.chat.openaiResponses']).toBe(original); }); it('folds the cache buckets into the input total when the totals witness it', async () => { // 100 + 50 + 10 = 160 reaches the stated total and 100 + 10 does not, so the upstream // reports the buckets outside the input count whatever anyone flagged. - const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); - const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.responses', { + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); + const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.openaiResponses', { 'route.attempt': attemptWith(), }, answer); - const frames = await drain(up['response.chat.responses']); + const frames = await drain(up['response.chat.openaiResponses']); expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 150, output_tokens: 10 }); }); @@ -950,47 +950,47 @@ describe('the Responses interceptors, as stages', () => { // input rather than the gate — and it arrives on the attempt this stage read on the way // down, so this is also what proves the closure carries it to the way back. it('folds on the flag alone when the response states no total', async () => { - const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, input_tokens_details: { cached_tokens: 50, cache_write_tokens: 5 } })]); - const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.responses', { + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, input_tokens_details: { cached_tokens: 50, cache_write_tokens: 5 } })]); + const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.openaiResponses', { 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), }, answer); - const frames = await drain(up['response.chat.responses']); + const frames = await drain(up['response.chat.openaiResponses']); expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 155 }); }); it('hands a frame back by identity when the totals say the buckets are already inside', async () => { const frame = usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 110, input_tokens_details: { cached_tokens: 50 } }); - const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.responses', { + const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.openaiResponses', { 'route.attempt': attemptWith(), - }, streamOf([frame])); - const frames = await drain(up['response.chat.responses']); + }, streamOf([frame])); + const frames = await drain(up['response.chat.openaiResponses']); expect(frames[0]).toBe(frame); }); it('leaves an answer that is not a stream alone, by identity', async () => { const answer = { kind: 'value' as const, body: null }; - const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.responses', { + const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.openaiResponses', { 'route.attempt': attemptWith(), }, answer); - expect(up['response.chat.responses']).toBe(answer); + expect(up['response.chat.openaiResponses']).toBe(answer); }); it('puts the reasoning sentinel into the shape DeepSeek reads', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForResponses, 'response.chat.responses', { - 'request.chat.responses': responsesPayload([], { reasoning: { effort: 'none' } }), + const { down } = await runStage(vendorDeepSeekNormalizeForResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([], { reasoning: { effort: 'none' } }), 'route.attempt': attemptWith('vendor-deepseek'), }); - const after = down['request.chat.responses'] as Record; + const after = down['request.chat.openaiResponses'] as Record; expect('reasoning' in after).toBe(false); expect(after.thinking).toEqual({ type: 'disabled' }); }); it('puts the reasoning sentinel into the shape Qwen reads', async () => { - const { down } = await runStage(vendorQwenNormalizeForResponses, 'response.chat.responses', { - 'request.chat.responses': responsesPayload([], { reasoning: { effort: 'none' } }), + const { down } = await runStage(vendorQwenNormalizeForResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([], { reasoning: { effort: 'none' } }), 'route.attempt': attemptWith('vendor-qwen'), }); - const after = down['request.chat.responses'] as Record; + const after = down['request.chat.openaiResponses'] as Record; expect('reasoning' in after).toBe(false); expect(after.enable_thinking).toBe(false); }); @@ -999,32 +999,32 @@ describe('the Responses interceptors, as stages', () => { // The dialect is the flag's, not the payload's: the same sentinel means something else on // the next upstream, and a rewrite that fired without the flag would put a Qwen field on a // wire that has never heard of it. - const original = responsesPayload([], { reasoning: { effort: 'none' } }); - const { down } = await runStage(vendorQwenNormalizeForResponses, 'response.chat.responses', { - 'request.chat.responses': original, + const original = openaiResponsesPayload([], { reasoning: { effort: 'none' } }); + const { down } = await runStage(vendorQwenNormalizeForResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': original, 'route.attempt': attemptWith(), }); - expect(down['request.chat.responses']).toBe(original); + expect(down['request.chat.openaiResponses']).toBe(original); }); it('leaves a payload that asked for real reasoning alone, by identity', async () => { - const original = responsesPayload([], { reasoning: { effort: 'high' } }); + const original = openaiResponsesPayload([], { reasoning: { effort: 'high' } }); for (const stage of [vendorDeepSeekNormalizeForResponses, vendorQwenNormalizeForResponses]) { - const { down } = await runStage(stage, 'response.chat.responses', { - 'request.chat.responses': original, + const { down } = await runStage(stage, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': original, 'route.attempt': attemptWith('vendor-deepseek', 'vendor-qwen'), }); - expect(down['request.chat.responses']).toBe(original); + expect(down['request.chat.openaiResponses']).toBe(original); } }); }); /** A lifecycle envelope carrying nothing but the counts under test. Every event that carries * a response resource repeats the whole resource, which is why the rewrite is per event. */ -const usageEvent = (usage: Record): ProtocolFrame => - eventFrame({ type: 'response.completed', sequence_number: 0, response: { usage } } as unknown as ResponsesStreamEvent); +const usageEvent = (usage: Record): ProtocolFrame => + eventFrame({ type: 'response.completed', sequence_number: 0, response: { usage } } as unknown as OpenAIResponsesStreamEvent); -const usageOf = (frame: ProtocolFrame): Record => +const usageOf = (frame: ProtocolFrame): Record => (frame as { event: { response: { usage: Record } } }).event.response.usage; // Each array below is the order a family's chain runs these rules in, and running it is what @@ -1036,12 +1036,12 @@ const usageOf = (frame: ProtocolFrame): Record { it('assembles and runs the Messages array', async () => { - const { down } = await runChain('response.chat.messages', [ + const { down } = await runChain('response.chat.anthropicMessages', [ stripBillingAttributionFromMessages, disableReasoningOnForcedToolChoiceForMessages, applyRoleCompatibilityToMessages, ], { - 'request.chat.messages': messagesPayload([{ role: 'system', content: 'inline' }], { + 'request.chat.anthropicMessages': anthropicMessagesPayload([{ role: 'system', content: 'inline' }], { system: 'x-anthropic-billing-header: cch=abcdef12\nreal instructions', tool_choice: { type: 'any' }, }), @@ -1051,34 +1051,34 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => 'rewrite-mid-conv-system-to-user', ), }); - const after = down['request.chat.messages'] as MessagesPayload; + const after = down['request.chat.anthropicMessages'] as AnthropicMessagesPayload; expect(after.system).toBe('real instructions'); expect(after.thinking).toEqual({ type: 'disabled' }); expect(after.messages.map(message => message.role)).toEqual(['user']); }); it('assembles and runs the Gemini array', async () => { - const answer = streamOf([ + const answer = streamOf([ eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }, { text: 'the answer' }] }, finishReason: 'STOP' }] }), ]); - const { down, up } = await runChain('response.chat.gemini', [ + const { down, up } = await runChain('response.chat.geminiGenerateContent', [ stripUnsupportedPartFieldsFromGemini, stripUnsupportedToolsFromGemini, stripSafetySettingsFromGemini, suppressThoughtPartsFromGemini, ], { - 'request.chat.gemini': { + 'request.chat.geminiGenerateContent': { contents: [{ role: 'user', parts: [{ text: 'hi', fileData: { mimeType: 'text/plain', fileUri: 'u' } }] }], tools: [{ googleSearch: {} }], safetySettings: [{ category: 'c', threshold: 't' }], - } satisfies GeminiPayload, + } satisfies GeminiGenerateContentPayload, }, answer); - const after = down['request.chat.gemini'] as GeminiPayload; + const after = down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload; expect(after.contents?.[0]?.parts).toEqual([{ text: 'hi' }]); expect('tools' in after).toBe(false); expect('safetySettings' in after).toBe(false); - const frames = await drain(up['response.chat.gemini']); - const event = (frames[0] as { event: Extract }).event; + const frames = await drain(up['response.chat.geminiGenerateContent']); + const event = (frames[0] as { event: Extract }).event; expect(event.candidates?.[0]?.content.parts).toEqual([{ text: 'the answer' }]); }); @@ -1086,8 +1086,8 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => // canonical form, and a vendor normalizer can only put it on the wire in the vendor's shape // because it runs after the stage that wrote it. it('assembles and runs the Responses array, the vendor normalizers last', async () => { - const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); - const { down, up } = await runChain('response.chat.responses', [ + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); + const { down, up } = await runChain('response.chat.openaiResponses', [ disableReasoningOnForcedToolChoiceForResponses, applyRoleCompatibilityToResponses, stripPromptCacheKeyForResponses, @@ -1095,7 +1095,7 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => vendorDeepSeekNormalizeForResponses, vendorQwenNormalizeForResponses, ], { - 'request.chat.responses': responsesPayload([ + 'request.chat.openaiResponses': openaiResponsesPayload([ { type: 'message', role: 'system', content: 'lead' }, { type: 'message', role: 'user', content: 'hi' }, { type: 'message', role: 'system', content: 'mid' }, @@ -1107,13 +1107,13 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => 'vendor-deepseek', ), }, answer); - const after = down['request.chat.responses'] as Record; + const after = down['request.chat.openaiResponses'] as Record; expect(rolesOfPayload(after)).toEqual(['system', 'user', 'user']); expect('prompt_cache_key' in after).toBe(false); // The sentinel this chain wrote reached DeepSeek's normalizer and left in DeepSeek's shape. expect('reasoning' in after).toBe(false); expect(after.thinking).toEqual({ type: 'disabled' }); - const frames = await drain(up['response.chat.responses']); + const frames = await drain(up['response.chat.openaiResponses']); expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 150 }); }); @@ -1123,7 +1123,7 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => // body. Coming back, the vendor dialect has the first say — so the fold reads a cache count // under OpenAI's name rather than DeepSeek's, and the carrier split reads the folded total. it('assembles and runs the Chat Completions wire, the vendor dialects innermost', async () => { - const answer = streamOf([eventFrame({ + const answer = streamOf([eventFrame({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], // DeepSeek's own names, and the exclusive convention: 479 + 13312 + 373 = 14164. @@ -1131,8 +1131,8 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_cache_hit_tokens: 13312, prompt_cache_miss_tokens: 479, }, - } as unknown as ChatCompletionsStreamEvent)]); - const { down, up } = await runChain('response.chat.chatCompletions', [ + } as unknown as OpenAIChatCompletionsStreamEvent)]); + const { down, up } = await runChain('response.chat.openaiChatCompletions', [ includeUsageStreamOptionsForChatCompletions, normalizeUsageForChatCompletions, applyRoleCompatibilityToChatCompletions, @@ -1141,7 +1141,7 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => vendorQwenNormalizeForChatCompletions, vendorKimiNormalizeForChatCompletions, ], { - 'request.chat.chatCompletions': chatCompletionsPayload([ + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ { role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }, { role: 'system', content: 'mid' }, @@ -1149,19 +1149,19 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user', 'vendor-deepseek'), }, answer); - const sent = down['request.chat.chatCompletions'] as Record; + const sent = down['request.chat.openaiChatCompletions'] as Record; expect(sent.stream_options).toEqual({ include_usage: true }); - expect((sent as unknown as ChatCompletionsPayload).messages.map(message => message.role)).toEqual(['system', 'user', 'user']); + expect((sent as unknown as OpenAIChatCompletionsPayload).messages.map(message => message.role)).toEqual(['system', 'user', 'user']); // The sentinel this chain carried reached DeepSeek's dialect and left in DeepSeek's shape. expect('reasoning_effort' in sent).toBe(false); expect(sent.thinking).toEqual({ type: 'disabled' }); - const frames = await drain(up['response.chat.chatCompletions']); + const frames = await drain(up['response.chat.openaiChatCompletions']); // Three rules ran on the way back, in the order that makes each one's input the one it // expects: the vendor rename, then the fold, then the split onto a carrier. expect(frames).toHaveLength(2); - expect('usage' in chatCompletionsEventOf(frames[0]!)).toBe(false); - expect(chatCompletionsUsageOf(frames[1]!)).toMatchObject({ + expect('usage' in openaiChatCompletionsEventOf(frames[0]!)).toBe(false); + expect(openaiChatCompletionsUsageOf(frames[1]!)).toMatchObject({ prompt_tokens: 13791, prompt_tokens_details: { cached_tokens: 13312 }, }); @@ -1169,4 +1169,4 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => }); const rolesOfPayload = (payload: Record): unknown[] => - (payload.input as ResponsesInputItem[]).map(item => (item as { role?: unknown }).role); + (payload.input as OpenAIResponsesInputItem[]).map(item => (item as { role?: unknown }).role); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts index 84f53f8f44..a79469aca2 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts @@ -5,7 +5,7 @@ import { test } from 'vitest'; -import { renderMessagesError } from '../../../../src/data-plane/chat/messages/errors.ts'; +import { renderMessagesError } from '../../../../src/data-plane/chat/anthropic-messages/errors.ts'; import { assert, assertEquals } from '@floway-dev/test-utils'; test('renders an Anthropic invalid_request_error envelope with a top-level request_id', () => { diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index 55d1a73eb8..b12962a14c 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -7,7 +7,7 @@ import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; import { flushBackground } from '../../../test-utils/background-tracker.ts'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; @@ -24,7 +24,7 @@ vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOrigin }; }); -const { messagesHttp } = await import('../../../../src/data-plane/chat/messages/http.ts'); +const { anthropicMessagesHttp } = await import('../../../../src/data-plane/chat/anthropic-messages/http.ts'); const API_KEY_ID = 'key_messages_http_test'; @@ -48,7 +48,7 @@ const buildApiKey = (overrides: Partial = {}): ApiKey => ({ upstreamIds: null, deletedAt: null, dumpRetentionSeconds: null, - responsesRetentionSeconds: 0, + openaiResponsesRetentionSeconds: 0, ...overrides, }); @@ -70,12 +70,12 @@ const makeApp = (): Hono<{ Variables: AuthVars }> => { c.set('user', buildUser()); await next(); }); - app.post('/v1/messages', messagesHttp.generate); - app.post('/v1/messages/count_tokens', messagesHttp.countTokens); + app.post('/v1/messages', anthropicMessagesHttp.generate); + app.post('/v1/messages/count_tokens', anthropicMessagesHttp.countTokens); return app; }; -const makeMessagesEvents = (): readonly MessagesStreamEvent[] => [ +const makeMessagesEvents = (): readonly AnthropicMessagesStreamEvent[] => [ { type: 'message_start', message: { @@ -104,7 +104,7 @@ const makeProtocolFrames = async function* (events: readonly TEvent[]): const makeCandidate = (overrides: { upstream?: string; endpoints?: ModelEndpoints; - callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; + callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; callMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; } = {}): ModelCandidate => { const upstream = overrides.upstream ?? 'up_test'; @@ -124,7 +124,7 @@ const makeCandidate = (overrides: { test('POST /v1/messages streams a successful SSE body', async () => { installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ + const callMessages = vi.fn(async (): Promise> => ({ ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers(), })); queueCandidates([makeCandidate({ callMessages })]); @@ -145,7 +145,7 @@ test('POST /v1/messages streams a successful SSE body', async () => { test('POST /v1/messages returns a single JSON body when stream is omitted', async () => { installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ + const callMessages = vi.fn(async (): Promise> => ({ ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers(), })); queueCandidates([makeCandidate({ callMessages })]); @@ -164,7 +164,7 @@ test('POST /v1/messages returns a single JSON body when stream is omitted', asyn test('POST /v1/messages answers the Claude Code model-validation probe without calling the upstream', async () => { const repo = installRepo(); - const callMessages = vi.fn((): Promise> => { + const callMessages = vi.fn((): Promise> => { throw new Error('the probe reached the upstream'); }); queueCandidates([makeCandidate({ callMessages })]); @@ -258,7 +258,7 @@ test('POST /v1/messages forwards upstream response headers end-to-end (streaming 'connection': 'close', 'set-cookie': 'session=secret', }); - const callMessages = vi.fn(async (): Promise> => ({ + const callMessages = vi.fn(async (): Promise> => ({ ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: upstreamHeaders, })); queueCandidates([makeCandidate({ callMessages })]); @@ -289,7 +289,7 @@ test('POST /v1/messages forwards upstream response headers end-to-end (non-strea 'anthropic-ratelimit-unified-status': 'allowed', 'cf-ray': 'cf_ray_e2e', }); - const callMessages = vi.fn(async (): Promise> => ({ + const callMessages = vi.fn(async (): Promise> => ({ ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: upstreamHeaders, })); queueCandidates([makeCandidate({ callMessages })]); @@ -308,7 +308,7 @@ test('POST /v1/messages forwards upstream response headers end-to-end (non-strea test('POST /v1/messages renders the Anthropic-shaped model-unsupported 400 when no candidate matches the messages-generate picker', async () => { installRepo(); // Queue a chat-kind candidate whose endpoints expose only `completions` — - // messagesGenerateTarget (messages > responses > chat-completions) rejects + // anthropicMessagesGenerateTarget (messages > responses > chat-completions) rejects // it, leaving zero viable candidates, and with sawModel=true the serve // renders model-unsupported as a 400. queueCandidates([makeCandidate({ endpoints: { completions: {} } })]); diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index 7785e2594f..d2f4a8d54a 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -6,12 +6,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; +import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import { CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, type ChatCompletionsPayload, type ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import { CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { SseFrame } from '@floway-dev/protocols/common'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -24,7 +24,7 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal let live: readonly ModelCandidate[] = []; const candidate = ( - callChatCompletions: (model: unknown, body: unknown) => Promise>, + callChatCompletions: (model: unknown, body: unknown) => Promise>, upstreamId = 'up_a', flags: readonly FlagId[] = [], ): ModelCandidate => { @@ -52,7 +52,7 @@ const resolves = (candidates: readonly ModelCandidate[]): void => { vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); }; -const chunk = (text: string): ChatCompletionsStreamEvent => ({ +const chunk = (text: string): OpenAIChatCompletionsStreamEvent => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', choices: [{ index: 0, delta: { content: text }, finish_reason: null }], }); @@ -61,11 +61,11 @@ const chunk = (text: string): ChatCompletionsStreamEvent => ({ * chunk whose delta holds nothing but `reasoning_opaque`. */ const affinityCarrier = (frame: SseFrame): string | undefined => { if (frame.data === '[DONE]') return undefined; - return (JSON.parse(frame.data) as ChatCompletionsStreamEvent).choices[0]?.delta.reasoning_opaque ?? undefined; + return (JSON.parse(frame.data) as OpenAIChatCompletionsStreamEvent).choices[0]?.delta.reasoning_opaque ?? undefined; }; /** A stream that stops without ever saying it ended — what a dropped upstream looks like. */ -const truncated = (...events: readonly ChatCompletionsStreamEvent[]): ProviderStreamResult => ({ +const truncated = (...events: readonly OpenAIChatCompletionsStreamEvent[]): ProviderStreamResult => ({ ok: true, modelKey: 'chat-model-key', headers: new Headers(), @@ -74,7 +74,7 @@ const truncated = (...events: readonly ChatCompletionsStreamEvent[]): ProviderSt })(), }); -const stream = (...events: readonly ChatCompletionsStreamEvent[]): ProviderStreamResult => ({ +const stream = (...events: readonly OpenAIChatCompletionsStreamEvent[]): ProviderStreamResult => ({ ok: true, modelKey: 'chat-model-key', headers: new Headers({ 'x-request-id': 'req-1', 'content-length': '99' }), @@ -84,11 +84,11 @@ const stream = (...events: readonly ChatCompletionsStreamEvent[]): ProviderStrea })(), }); -const payload = { model: 'chat-model', messages: [{ role: 'user', content: 'hi' }] } as unknown as ChatCompletionsPayload; +const payload = { model: 'chat-model', messages: [{ role: 'user', content: 'hi' }] } as unknown as OpenAIChatCompletionsPayload; /** What affinity materialized for the candidate about to be dialled. It differs from the * payload the client sent, which is the point: carried state is rewritten per candidate. */ -let affinityPayload: ChatCompletionsPayload = payload; +let affinityPayload: OpenAIChatCompletionsPayload = payload; const serve = async (wantsStream: boolean) => await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream); @@ -97,13 +97,13 @@ const serveWith = async ( wantsStream: boolean, wantsUsageChunk = false, ) => await run( - chatCompletionsServePipeline(payload), + openaiChatCompletionsServePipeline(payload), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], 'ingress.chat.sourceProtocol': 'chatCompletions', - 'ingress.chat.chatCompletions.wantsStream': wantsStream, - 'ingress.chat.chatCompletions.wantsUsageChunk': wantsUsageChunk, - 'request.chat.chatCompletions': payload, + 'ingress.chat.openaiChatCompletions.wantsStream': wantsStream, + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': wantsUsageChunk, + 'request.chat.openaiChatCompletions': payload, 'serve.model': 'chat-model', }) as never, { @@ -140,7 +140,7 @@ describe('the chat completions chain', () => { affinityPayload = { ...payload, messages: [{ role: 'system', content: 'be brief' }, { role: 'user', content: 'hi' }], - } as unknown as ChatCompletionsPayload; + } as unknown as OpenAIChatCompletionsPayload; resolves([candidate(async (_model, body) => { sent = body as { messages: { role: string }[] }; return stream(chunk('hi')); @@ -156,7 +156,7 @@ describe('the chat completions chain', () => { // that goes out is the one it materialized rather than the one the client sent. it('sends the payload affinity materialized for the candidate it dialled', async () => { let sent: Record | undefined; - affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as ChatCompletionsPayload; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as OpenAIChatCompletionsPayload; resolves([candidate(async (_model, body) => { sent = body as Record; return stream(chunk('hi')); @@ -174,7 +174,7 @@ describe('the chat completions chain', () => { const { facts } = await serve(true); const frames: SseFrame[] = []; - for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) frames.push(frame); expect(facts['response.http.status']).toBe(200); // The carrier the edge writes has a test of its own below; what the protocol itself @@ -195,7 +195,7 @@ describe('the chat completions chain', () => { const { facts } = await serveWith(gateway, true); const frames: SseFrame[] = []; - for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) frames.push(frame); const carried = frames.flatMap(frame => affinityCarrier(frame) ?? []); expect(carried).toHaveLength(1); @@ -213,7 +213,7 @@ describe('the chat completions chain', () => { resolves([candidate(async () => stream(chunk('he'), chunk('llo')))]); const { facts } = await serve(false); - const rendered = facts['response.chat.chatCompletions.rendered'] as { choices: { message: { content: string } }[] }; + const rendered = facts['response.chat.openaiChatCompletions.rendered'] as { choices: { message: { content: string } }[] }; expect(facts['response.http.status']).toBe(200); expect(rendered.choices[0]!.message.content).toBe('hello'); @@ -242,7 +242,7 @@ describe('the chat completions chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(429); - expect(facts['response.chat.chatCompletions.rendered']).toEqual({ error: { message: 'slow down' } }); + expect(facts['response.chat.openaiChatCompletions.rendered']).toEqual({ error: { message: 'slow down' } }); }); // What the upstream reported is per token category and what is billed is per metric name; @@ -251,11 +251,11 @@ describe('the chat completions chain', () => { resolves([candidate(async () => stream(chunk('hi'), { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', choices: [], usage: { prompt_tokens: 11, completion_tokens: 4, total_tokens: 15 }, - } as unknown as ChatCompletionsStreamEvent))]); + } as unknown as OpenAIChatCompletionsStreamEvent))]); const { facts } = await serve(true); - for await (const _frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { /* drain */ } - const outcome = await facts['response.chat.chatCompletions.streamedUsage']!; + for await (const _frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { /* drain */ } + const outcome = await facts['response.chat.openaiChatCompletions.streamedUsage']!; expect(outcome.billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); // A rate can depend on the tier and on how much input there was; both are selector @@ -278,7 +278,7 @@ describe('the chat completions chain', () => { ? stream(chunk('hi'), { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', choices: [], usage: { prompt_tokens: 11, completion_tokens: 4, total_tokens: 15 }, - } as unknown as ChatCompletionsStreamEvent) + } as unknown as OpenAIChatCompletionsStreamEvent) : stream(chunk('hi')); })]); @@ -286,8 +286,8 @@ describe('the chat completions chain', () => { // the two questions come apart: the upstream is asked for it regardless. const { facts } = await serve(true); const frames: SseFrame[] = []; - for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); - const outcome = await facts['response.chat.chatCompletions.streamedUsage']!; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) frames.push(frame); + const outcome = await facts['response.chat.openaiChatCompletions.streamedUsage']!; expect(asked).toEqual({ include_usage: true }); expect(outcome.billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); @@ -296,7 +296,7 @@ describe('the chat completions chain', () => { }); // The wire's response-direction rules, on the wire that owns them, in the order it composes - // them — which is a property of the array in `chatCompletionsWire` and of nothing else. A + // them — which is a property of the array in `openaiChatCompletionsWire` and of nothing else. A // DeepSeek-shaped upstream reporting the exclusive cache convention exercises all three at // once, and each one's input is what the one below it left: the vendor rename, then the fold // that reads the renamed field, then the split that puts the settled usage on a carrier. @@ -305,7 +305,7 @@ describe('the chat completions chain', () => { { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', choices: [{ index: 0, delta: { reasoning_content: 'pondering' }, finish_reason: null }], - } as unknown as ChatCompletionsStreamEvent, + } as unknown as OpenAIChatCompletionsStreamEvent, { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', // Hung off the final delta rather than sent on a carrier, in DeepSeek's own cache @@ -316,14 +316,14 @@ describe('the chat completions chain', () => { prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_cache_hit_tokens: 13312, prompt_cache_miss_tokens: 479, }, - } as unknown as ChatCompletionsStreamEvent, + } as unknown as OpenAIChatCompletionsStreamEvent, ), 'up_a', ['vendor-deepseek'])]); const { facts } = await serveWith(mockChatGatewayCtx({ wantsStream: true }), true, true); - const written: ChatCompletionsStreamEvent[] = []; - for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { + const written: OpenAIChatCompletionsStreamEvent[] = []; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { if (frame.data !== '[DONE]' && affinityCarrier(frame) === undefined) { - written.push(JSON.parse(frame.data) as ChatCompletionsStreamEvent); + written.push(JSON.parse(frame.data) as OpenAIChatCompletionsStreamEvent); } } @@ -352,20 +352,20 @@ describe('the chat completions chain', () => { // carries one of them. it('serves the request-side and response-side vendor dialects the wire also composes', async () => { let sent: Record | undefined; - affinityPayload = { ...payload, reasoning_effort: 'none' } as ChatCompletionsPayload; + affinityPayload = { ...payload, reasoning_effort: 'none' } as OpenAIChatCompletionsPayload; resolves([candidate(async (_model, body) => { sent = body as Record; return stream(chunk('hi'), { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', choices: [], usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, cached_tokens: 30 }, - } as unknown as ChatCompletionsStreamEvent); + } as unknown as OpenAIChatCompletionsStreamEvent); }, 'up_a', ['vendor-qwen', 'vendor-kimi'])]); const { facts } = await serveWith(mockChatGatewayCtx({ wantsStream: true }), true, true); - const written: ChatCompletionsStreamEvent[] = []; - for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { + const written: OpenAIChatCompletionsStreamEvent[] = []; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { if (frame.data !== '[DONE]' && affinityCarrier(frame) === undefined) { - written.push(JSON.parse(frame.data) as ChatCompletionsStreamEvent); + written.push(JSON.parse(frame.data) as OpenAIChatCompletionsStreamEvent); } } affinityPayload = payload; @@ -392,7 +392,7 @@ describe('the chat completions chain', () => { const { facts } = await serveWith(gateway, true); expect(gateway.attempt.firstOutputTokenAt).toBeNull(); - for await (const _frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { /* drain */ } + for await (const _frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { /* drain */ } expect(gateway.attempt.firstOutputTokenAt).toBeTypeOf('number'); }); @@ -406,7 +406,7 @@ describe('the chat completions chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(400); - expect(facts['response.chat.chatCompletions.rendered']).toEqual({ + expect(facts['response.chat.openaiChatCompletions.rendered']).toEqual({ error: { message: 'Model chat-model does not support the /chat/completions endpoint.', type: 'invalid_request_error' }, }); }); @@ -419,7 +419,7 @@ describe('the chat completions chain', () => { const { facts } = await serve(true); const drain = async (): Promise => { - for await (const _frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) { /* to the end */ } + for await (const _frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { /* to the end */ } }; await expect(drain()).rejects.toThrow(CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE); @@ -431,7 +431,7 @@ describe('the chat completions chain', () => { const { facts } = await serve(true); const frames: SseFrame[] = []; - for await (const frame of facts['response.chat.chatCompletions.rendered'] as AsyncIterable) frames.push(frame); + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) frames.push(frame); expect(frames.at(-1)!.data).toBe('[DONE]'); }); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts index a4eb9bd098..682a5f0dd2 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts @@ -5,15 +5,15 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { geminiCountTokensPipeline } from '../../../src/data-plane/chat/gemini/count-tokens.ts'; +import { geminiGenerateContentCountTokensPipeline } from '../../../src/data-plane/chat/gemini-generate-content/count-tokens.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { ModelEndpoints } from '@floway-dev/protocols/common'; -import type { GeminiPayload } from '@floway-dev/protocols/gemini'; -import type { MessagesPayload } from '@floway-dev/protocols/messages'; -import type { MessagesUpstreamCallOptions, ModelCandidate, ProviderCallResult } from '@floway-dev/provider'; +import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { AnthropicMessagesUpstreamCallOptions, ModelCandidate, ProviderCallResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -27,7 +27,7 @@ type CountTokens = ( model: unknown, body: unknown, signal: AbortSignal | undefined, - opts: MessagesUpstreamCallOptions, + opts: AnthropicMessagesUpstreamCallOptions, ) => Promise; const candidate = ( @@ -67,19 +67,19 @@ const answered = (body: unknown): ProviderCallResult => ({ modelKey: 'claude-model-key', }); -const payload: GeminiPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; +const payload: GeminiGenerateContentPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; const count = async ( - request: GeminiPayload = payload, + request: GeminiGenerateContentPayload = payload, headers: readonly (readonly [string, string])[] = [], ) => { const gateway = mockChatGatewayCtx({ wantsStream: false }); return await run( - geminiCountTokensPipeline(request), + geminiGenerateContentCountTokensPipeline(request), move({ 'ingress.http.headers': headers, 'ingress.chat.sourceProtocol': 'gemini', - 'request.chat.gemini': request, + 'request.chat.geminiGenerateContent': request, 'serve.model': 'gemini-model', }) as never, { @@ -110,18 +110,18 @@ describe('the gemini count-tokens chain', () => { // Gemini has no endpoint that answers this, so the question is asked in Messages and the // counts are read back out under the name this protocol gives them. it('asks the question in Messages and answers in Google-s own envelope', async () => { - let sent: MessagesPayload | undefined; + let sent: AnthropicMessagesPayload | undefined; let seenModel: { readonly id: string } | undefined; resolves([candidate(async (model, body) => { seenModel = model as { readonly id: string }; - sent = body as MessagesPayload; + sent = body as AnthropicMessagesPayload; return answered({ input_tokens: 42 }); })]); const { facts } = await count({ ...payload, systemInstruction: { parts: [{ text: 'system' }] } }); expect(facts['response.http.status']).toBe(200); - expect(facts['response.chat.gemini.rendered']).toEqual({ totalTokens: 42 }); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ totalTokens: 42 }); expect(seenModel?.id).toBe('claude-model'); expect(sent?.system).toBeDefined(); expect(sent?.messages).toMatchObject([{ role: 'user', content: [{ type: 'text', text: 'hi' }] }]); @@ -136,7 +136,7 @@ describe('the gemini count-tokens chain', () => { const { facts } = await count(); - expect(facts['response.chat.gemini.rendered']).toEqual({ totalTokens: 19 }); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ totalTokens: 19 }); }); // A body carrying neither figure is an answer this gateway cannot read, which is the @@ -153,7 +153,7 @@ describe('the gemini count-tokens chain', () => { expect(tried).toEqual(['odd', 'odder']); expect(facts['response.http.status']).toBe(502); - expect(facts['response.chat.gemini.rendered']).toEqual({ + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ error: { code: 502, message: 'Invalid upstream token counting response.', status: 'UNAVAILABLE' }, }); }); @@ -162,16 +162,16 @@ describe('the gemini count-tokens chain', () => { // candidate answers — and a count taken before the strip would measure a request the // translator would refuse. it('strips what no translation carries before asking', async () => { - let sent: MessagesPayload | undefined; - resolves([candidate(async (_model, body) => { sent = body as MessagesPayload; return answered({ input_tokens: 5 }); })]); + let sent: AnthropicMessagesPayload | undefined; + resolves([candidate(async (_model, body) => { sent = body as AnthropicMessagesPayload; return answered({ input_tokens: 5 }); })]); const { facts } = await count({ contents: [{ role: 'user', parts: [{ text: 'hi' }, { fileData: { fileUri: 'gs://x', mimeType: 'text/plain' } }] }], tools: [{ googleSearch: {} }], safetySettings: [{ category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_NONE' }], - } as unknown as GeminiPayload); + } as unknown as GeminiGenerateContentPayload); - expect(facts['response.chat.gemini.rendered']).toEqual({ totalTokens: 5 }); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ totalTokens: 5 }); // A tool group left declaring no function is not a tool group, so `tools` goes with it. expect(sent).not.toHaveProperty('tools'); expect(JSON.stringify(sent)).not.toContain('fileData'); @@ -181,7 +181,7 @@ describe('the gemini count-tokens chain', () => { // Anthropic's beta flags are the client's own only when the client spoke that protocol, and // a Gemini turn asked for nothing on that wire. it('does not carry another protocol-s beta flags across the boundary', async () => { - let seen: MessagesUpstreamCallOptions | undefined; + let seen: AnthropicMessagesUpstreamCallOptions | undefined; resolves([candidate(async (_model, _body, _signal, opts) => { seen = opts; return answered({ input_tokens: 1 }); })]); await count(payload, [['anthropic-beta', 'must-not-cross-source-protocols'], ['x-trace', 'abc']]); @@ -197,7 +197,7 @@ describe('the gemini count-tokens chain', () => { const { facts } = await count(); expect(facts['response.http.status']).toBe(503); - expect(facts['response.chat.gemini.rendered']).toEqual({ + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ error: { code: 503, message: 'Upstream token counting request failed.', status: 'UNAVAILABLE' }, }); }); @@ -210,7 +210,7 @@ describe('the gemini count-tokens chain', () => { const { facts } = await count(); expect(facts['response.http.status']).toBe(400); - expect(facts['response.chat.gemini.rendered']).toEqual({ + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ error: { code: 400, message: 'Model gemini-model does not support countTokens.', status: 'INVALID_ARGUMENT' }, }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index 70e47a51ea..a979f0f85a 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -6,14 +6,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; +import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import type { ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; -import { GEMINI_MISSING_TERMINAL_MESSAGE, type GeminiPayload, type GeminiResult } from '@floway-dev/protocols/gemini'; +import { GEMINI_MISSING_TERMINAL_MESSAGE, type GeminiGenerateContentPayload, type GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; import { directFetcher, type ModelCandidate, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -43,7 +43,7 @@ type CallChatCompletions = ( body: unknown, signal: AbortSignal | undefined, opts: UpstreamCallOptions, -) => Promise>; +) => Promise>; const candidate = (upstream: string, callChatCompletions: CallChatCompletions): ModelCandidate => { const endpoints = { chatCompletions: {} }; @@ -58,21 +58,21 @@ const candidate = (upstream: string, callChatCompletions: CallChatCompletions): }; }; -const chunk = (delta: Record, finishReason?: string): ProtocolFrame => +const chunk = (delta: Record, finishReason?: string): ProtocolFrame => eventFrame({ id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'gemini-2.5-pro', choices: [{ index: 0, delta, ...(finishReason === undefined ? {} : { finish_reason: finishReason }) }], - } as ChatCompletionsStreamEvent); + } as OpenAIChatCompletionsStreamEvent); -const usageChunk: ProtocolFrame = eventFrame({ +const usageChunk: ProtocolFrame = eventFrame({ id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'gemini-2.5-pro', choices: [], usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 }, -} as ChatCompletionsStreamEvent); +} as OpenAIChatCompletionsStreamEvent); /** One whole turn on the wire below: two content deltas, the figures the upstream metered, * and the sentinel that ends it. */ -const turn = (): AsyncIterable> => ({ +const turn = (): AsyncIterable> => ({ [Symbol.asyncIterator]: () => (async function* () { yield chunk({ content: 'he' }); yield chunk({ content: 'llo' }, 'stop'); @@ -83,7 +83,7 @@ const turn = (): AsyncIterable> => ({ /** The wire below closed cleanly and never finished the turn: no choice ever carried a * finish reason, so nothing that comes out of the translation is a Gemini terminal event. */ -const unfinishedTurn = (): AsyncIterable> => ({ +const unfinishedTurn = (): AsyncIterable> => ({ [Symbol.asyncIterator]: () => (async function* () { yield chunk({ content: 'he' }); yield doneFrame(); @@ -91,9 +91,9 @@ const unfinishedTurn = (): AsyncIterable>, + events: AsyncIterable>, headers?: Headers, -): ProviderStreamResult => ({ +): ProviderStreamResult => ({ ok: true, events, modelKey: 'gemini-2.5-pro-key', ...(headers === undefined ? {} : { headers }), }); @@ -101,7 +101,7 @@ const payload = { contents: [{ role: 'user' as const, parts: [{ text: 'hello' }] /** What affinity materialized for the candidate about to be dialled. It differs from the * turn the client sent, which is the point: carried state is rewritten per candidate. */ -let affinityPayload: GeminiPayload = payload; +let affinityPayload: GeminiGenerateContentPayload = payload; /** How many times the chain asked the resolver for that turn. One stage owns the reading; * anything below it reads the record. */ @@ -131,10 +131,10 @@ beforeEach(() => { }); const serve = async (facts: Record) => - await serveWith(mockChatGatewayCtx({ wantsStream: facts['ingress.chat.gemini.wantsStream'] === true }), facts); + await serveWith(mockChatGatewayCtx({ wantsStream: facts['ingress.chat.geminiGenerateContent.wantsStream'] === true }), facts); const serveWith = async (gateway: ReturnType, facts: Record) => await run( - geminiServePipeline(payload), + geminiGenerateContentServePipeline(payload), move(facts) as never, { gateway, @@ -150,10 +150,10 @@ const serveWith = async (gateway: ReturnType, facts: ); const entryFacts = (overrides: Record = {}) => ({ - 'ingress.chat.gemini.wantsStream': true, + 'ingress.chat.geminiGenerateContent.wantsStream': true, 'ingress.chat.sourceProtocol': 'gemini', 'ingress.http.headers': [], - 'request.chat.gemini': payload, + 'request.chat.geminiGenerateContent': payload, 'serve.model': 'gemini-2.5-pro', ...overrides, }); @@ -167,7 +167,7 @@ const collect = async (rendered: unknown): Promise => { /** The turn's own answer. The carrier the edge writes back rides as a `thoughtSignature` on * a Part rather than as an event of its own, so taking it off leaves what Gemini itself * said. */ -const withoutCarrier = (event: GeminiResult): GeminiResult => ({ +const withoutCarrier = (event: GeminiGenerateContentResult): GeminiGenerateContentResult => ({ ...event, ...(event.candidates === undefined ? {} : { candidates: event.candidates.map(candidate => ({ @@ -181,19 +181,19 @@ const withoutCarrier = (event: GeminiResult): GeminiResult => ({ }); /** Every piece of this turn's own state the client was handed back. */ -const affinityCarriers = (events: readonly GeminiResult[]): readonly string[] => +const affinityCarriers = (events: readonly GeminiGenerateContentResult[]): readonly string[] => events.flatMap(event => (event.candidates ?? []).flatMap(candidate => candidate.content.parts.flatMap(part => part.thoughtSignature ?? []))); describe('the gemini pipeline', () => { - // The whole entry contract, and it does not mention `request.chat.gemini`: the turn the + // The whole entry contract, and it does not mention `request.chat.geminiGenerateContent`: the turn the // wire translates is not the one the caller handed in but the one materializeAttempt put // into the record below the fork, which is where a per-candidate payload can exist at all. // The two `ingress.*` keys are there because the fork declares what the wire under it // reads — a wire is built against a candidate, so assembly cannot ask one itself. it('assembles, and asks its caller only for what the descending stages need', () => { - expect([...geminiServePipeline(payload).entryNeeds].sort()).toEqual([ - 'ingress.chat.gemini.wantsStream', + expect([...geminiGenerateContentServePipeline(payload).entryNeeds].sort()).toEqual([ + 'ingress.chat.geminiGenerateContent.wantsStream', 'ingress.chat.sourceProtocol', 'ingress.http.headers', 'serve.model', @@ -211,8 +211,8 @@ describe('the gemini pipeline', () => { // The answer comes back before the drain runs, which is what lets a streaming family // hand its stream on: the frames are still there to read. - expect((await collect(facts['response.chat.gemini.rendered'])) - .map(frame => withoutCarrier(JSON.parse(frame.data) as GeminiResult))).toEqual([ + expect((await collect(facts['response.chat.geminiGenerateContent.rendered'])) + .map(frame => withoutCarrier(JSON.parse(frame.data) as GeminiGenerateContentResult))).toEqual([ { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'he' }] } }] }, { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'llo' }] }, finishReason: 'STOP' }], @@ -242,13 +242,13 @@ describe('the gemini pipeline', () => { resolves([dialled]); const { facts, drain } = await serve(entryFacts()); - await collect(facts['response.chat.gemini.rendered']); + await collect(facts['response.chat.geminiGenerateContent.rendered']); - // The handoff consumed `request.chat.gemini` and provided the wire's own key, so what the + // The handoff consumed `request.chat.geminiGenerateContent` and provided the wire's own key, so what the // record still carries below the fork is the translated turn and not the Gemini one — the // whole of what a translation is, said in the fact space. - expect((facts as Record)['request.chat.gemini']).toBeUndefined(); - expect((facts as Record)['request.chat.chatCompletions']) + expect((facts as Record)['request.chat.geminiGenerateContent']).toBeUndefined(); + expect((facts as Record)['request.chat.openaiChatCompletions']) .toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); expect(asked).toBe(1); expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); @@ -259,11 +259,11 @@ describe('the gemini pipeline', () => { it('folds the frames into one Gemini result for a client that did not ask to stream', async () => { resolves([candidate('up_a', async () => streamed(turn()))]); - const { facts, drain } = await serve(entryFacts({ 'ingress.chat.gemini.wantsStream': false })); + const { facts, drain } = await serve(entryFacts({ 'ingress.chat.geminiGenerateContent.wantsStream': false })); // The two chunks stay two Parts: the carrier signs the Part it rides on, and merging a // signed Part into its neighbour would spread that signature over text it does not cover. - expect(withoutCarrier(facts['response.chat.gemini.rendered'] as GeminiResult)).toEqual({ + expect(withoutCarrier(facts['response.chat.geminiGenerateContent.rendered'] as GeminiGenerateContentResult)).toEqual({ candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'he' }, { text: 'llo' }] }, finishReason: 'STOP' }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 7, totalTokenCount: 12 }, }); @@ -279,8 +279,8 @@ describe('the gemini pipeline', () => { const gateway = mockChatGatewayCtx({ wantsStream: true }); const { facts, drain } = await serveWith(gateway, entryFacts()); - const frames = await collect(facts['response.chat.gemini.rendered']); - const carried = affinityCarriers(frames.map(frame => JSON.parse(frame.data) as GeminiResult)); + const frames = await collect(facts['response.chat.geminiGenerateContent.rendered']); + const carried = affinityCarriers(frames.map(frame => JSON.parse(frame.data) as GeminiGenerateContentResult)); expect(carried).toHaveLength(1); // It names the upstream that answered, sealed under this run's own secret — which is what @@ -313,7 +313,7 @@ describe('the gemini pipeline', () => { // because every Gemini wire is a translation and the object that came back was written by // whatever protocol the candidate was dialled on. expect(facts['response.http.status']).toBe(400); - const rendered = facts['response.chat.gemini.rendered'] as { error: { code: number; message: string; status: string } }; + const rendered = facts['response.chat.geminiGenerateContent.rendered'] as { error: { code: number; message: string; status: string } }; expect(rendered.error.code).toBe(400); expect(rendered.error.status).toBe('INVALID_ARGUMENT'); expect(rendered.error.message).toContain('no'); @@ -331,7 +331,7 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts()); expect(facts['response.http.status']).toBe(502); - expect(facts['response.chat.gemini.rendered']).toEqual({ error: { code: 502, message: 'socket hang up', status: 'UNAVAILABLE' } }); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ error: { code: 502, message: 'socket hang up', status: 'UNAVAILABLE' } }); // A dial that never completed reached no upstream, so nothing was billed and there are no // headers to carry. expect(facts['response.usage.billable']).toEqual([]); @@ -349,7 +349,7 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts()); expect(facts['response.http.headers']).toEqual([['x-request-id', 'req_1']]); - await collect(facts['response.chat.gemini.rendered']); + await collect(facts['response.chat.geminiGenerateContent.rendered']); await drain(); }); @@ -361,11 +361,11 @@ describe('the gemini pipeline', () => { resolves([candidate('up_a', async () => streamed(turn()))]); const { facts, drain } = await serve(entryFacts()); - await collect(facts['response.chat.gemini.rendered']); + await collect(facts['response.chat.geminiGenerateContent.rendered']); await drain(); expect(recorded.usage).toHaveLength(0); - const usage = facts['response.chat.gemini.streamedUsage']; + const usage = facts['response.chat.geminiGenerateContent.streamedUsage']; expect(usage).not.toBeNull(); const outcome = await usage!; const billable = outcome.billable; @@ -387,7 +387,7 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serveWith(gateway, entryFacts()); expect(gateway.attempt.firstOutputTokenAt).toBeNull(); - await collect(facts['response.chat.gemini.rendered']); + await collect(facts['response.chat.geminiGenerateContent.rendered']); expect(gateway.attempt.firstOutputTokenAt).toBeTypeOf('number'); await drain(); @@ -401,7 +401,7 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts()); - await expect(collect(facts['response.chat.gemini.rendered'])).rejects.toThrow(GEMINI_MISSING_TERMINAL_MESSAGE); + await expect(collect(facts['response.chat.geminiGenerateContent.rendered'])).rejects.toThrow(GEMINI_MISSING_TERMINAL_MESSAGE); await drain(); }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts index 3d48a5c86e..b22fecea8e 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts @@ -6,14 +6,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { messagesCountTokensPipeline } from '../../../src/data-plane/chat/messages/count-tokens.ts'; +import { anthropicMessagesCountTokensPipeline } from '../../../src/data-plane/chat/anthropic-messages/count-tokens.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { AliasRules, ModelEndpoints } from '@floway-dev/protocols/common'; -import type { MessagesPayload } from '@floway-dev/protocols/messages'; -import { type FlagId, type MessagesUpstreamCallOptions, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { type FlagId, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -27,7 +27,7 @@ type CountTokens = ( model: unknown, body: unknown, signal: AbortSignal | undefined, - opts: MessagesUpstreamCallOptions, + opts: AnthropicMessagesUpstreamCallOptions, ) => Promise; const candidate = ( @@ -73,23 +73,23 @@ const counted = (input: number): ProviderCallResult => ({ modelKey: 'claude-model-key', }); -const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as MessagesPayload; +const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as AnthropicMessagesPayload; /** What affinity materialized for the candidate about to be dialled. It differs from the * payload the client sent, which is the point: carried state is rewritten per candidate. */ -let affinityPayload: MessagesPayload = payload; +let affinityPayload: AnthropicMessagesPayload = payload; const count = async ( - request: MessagesPayload = payload, + request: AnthropicMessagesPayload = payload, headers: readonly (readonly [string, string])[] = [], ) => { const gateway = mockChatGatewayCtx({ wantsStream: false }); return await run( - messagesCountTokensPipeline(request), + anthropicMessagesCountTokensPipeline(request), move({ 'ingress.http.headers': headers, 'ingress.chat.sourceProtocol': 'messages', - 'request.chat.messages': request, + 'request.chat.anthropicMessages': request, 'serve.model': request.model, }) as never, { @@ -124,7 +124,7 @@ describe('the messages count-tokens chain', () => { it('measures the payload this attempt is owed, under the id the upstream resolved', async () => { let sent: Record | undefined; let seenModel: { readonly id: string } | undefined; - affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as MessagesPayload; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as AnthropicMessagesPayload; resolves([candidate(async (model, body) => { seenModel = model as { readonly id: string }; sent = body as Record; @@ -134,7 +134,7 @@ describe('the messages count-tokens chain', () => { const { facts } = await count(); expect(facts['response.http.status']).toBe(200); - expect(facts['response.chat.messages.rendered']).toEqual({ input_tokens: 11 }); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ input_tokens: 11 }); expect(seenModel?.id).toBe('claude-model'); expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); // The id the client addressed does not travel: the provider re-stamps what it resolved. @@ -158,7 +158,7 @@ describe('the messages count-tokens chain', () => { thinking: { type: 'enabled', budget_tokens: 1024 }, output_config: { effort: 'high' }, tool_choice: { type: 'tool', name: 'lookup' }, - } as unknown as MessagesPayload; + } as unknown as AnthropicMessagesPayload; resolves([candidate(async (_model, body) => { sent = body as Record; return counted(9); }, { enabledFlags: new Set([ 'strip-billing-attribution', @@ -185,10 +185,10 @@ describe('the messages count-tokens chain', () => { // ordinary client tool — so a count taken on the client's own body would measure a request // nobody is sent. it('measures the client-tool shape the web-search shim would actually send', async () => { - let sent: MessagesPayload | undefined; - const asked = { ...payload, tools: [{ type: 'web_search_20260209', max_uses: 3 }] } as unknown as MessagesPayload; + let sent: AnthropicMessagesPayload | undefined; + const asked = { ...payload, tools: [{ type: 'web_search_20260209', max_uses: 3 }] } as unknown as AnthropicMessagesPayload; affinityPayload = asked; - resolves([candidate(async (_model, body) => { sent = body as MessagesPayload; return counted(13); }, { + resolves([candidate(async (_model, body) => { sent = body as AnthropicMessagesPayload; return counted(13); }, { enabledFlags: new Set(['messages-web-search-shim']), })]); @@ -213,7 +213,7 @@ describe('the messages count-tokens chain', () => { { type: 'web_search_20260209' }, { type: 'web_search_20250305' }, ], - } as unknown as MessagesPayload; + } as unknown as AnthropicMessagesPayload; affinityPayload = asked; let dialled = 0; resolves([candidate(async () => { dialled += 1; return counted(0); }, { @@ -224,7 +224,7 @@ describe('the messages count-tokens chain', () => { expect(dialled).toBe(0); expect(facts['response.http.status']).toBe(400); - expect(facts['response.chat.messages.rendered']).toMatchObject({ + expect(facts['response.chat.anthropicMessages.rendered']).toMatchObject({ type: 'error', error: { type: 'invalid_request_error', message: 'Only one native web search tool definition is supported per request.' }, }); @@ -233,10 +233,10 @@ describe('the messages count-tokens chain', () => { // The flag is the whole of the gate here: counting has only the native wire, so the // structural half of the shim's condition — a translated target — can never hold. it('leaves the native tool alone on a candidate the shim is off for', async () => { - let sent: MessagesPayload | undefined; - const asked = { ...payload, tools: [{ type: 'web_search_20260209', max_uses: 3 }] } as unknown as MessagesPayload; + let sent: AnthropicMessagesPayload | undefined; + const asked = { ...payload, tools: [{ type: 'web_search_20260209', max_uses: 3 }] } as unknown as AnthropicMessagesPayload; affinityPayload = asked; - resolves([candidate(async (_model, body) => { sent = body as MessagesPayload; return counted(13); })]); + resolves([candidate(async (_model, body) => { sent = body as AnthropicMessagesPayload; return counted(13); })]); await count(asked); @@ -246,7 +246,7 @@ describe('the messages count-tokens chain', () => { // Anthropic beta flags have a typed path of their own precisely so no provider's header // allowlist can admit them, and a measurement asks the same question generation does. it('hands the beta flags over on their own path and not as a header', async () => { - let seen: MessagesUpstreamCallOptions | undefined; + let seen: AnthropicMessagesUpstreamCallOptions | undefined; resolves([candidate(async (_model, _body, _signal, opts) => { seen = opts; return counted(5); })]); await count(payload, [ @@ -275,7 +275,7 @@ describe('the messages count-tokens chain', () => { expect(tried).toEqual(['busy', 'free']); expect(facts['response.http.status']).toBe(200); - expect(facts['response.chat.messages.rendered']).toEqual({ input_tokens: 42 }); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ input_tokens: 42 }); }); it('answers the last refusal in the upstream-s own status and words', async () => { @@ -289,7 +289,7 @@ describe('the messages count-tokens chain', () => { const { facts } = await count(); expect(facts['response.http.status']).toBe(429); - expect(facts['response.chat.messages.rendered']).toEqual({ + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ type: 'error', error: { type: 'rate_limit_error', message: 'slow down' }, }); @@ -307,7 +307,7 @@ describe('the messages count-tokens chain', () => { const { facts } = await count(); expect(tried).toEqual(['dead', 'alive']); - expect(facts['response.chat.messages.rendered']).toEqual({ input_tokens: 7 }); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ input_tokens: 7 }); }); // Only a native Messages endpoint measures — no translation carries the question — so a @@ -319,7 +319,7 @@ describe('the messages count-tokens chain', () => { const { facts } = await count(); expect(facts['response.http.status']).toBe(400); - expect(facts['response.chat.messages.rendered']).toMatchObject({ + expect(facts['response.chat.anthropicMessages.rendered']).toMatchObject({ error: { type: 'invalid_request_error', message: 'Model claude-model does not support the /messages/count_tokens endpoint.', diff --git a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts index e9bd8ed991..e5276912e0 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts @@ -8,14 +8,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { messagesServePipeline } from '../../../src/data-plane/chat/messages/pipeline.ts'; +import { anthropicMessagesServePipeline } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { MESSAGES_MISSING_TERMINAL_MESSAGE, type MessagesPayload, type MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import { directFetcher, type MessagesUpstreamCallOptions, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; +import { MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { directFetcher, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -29,8 +29,8 @@ type CallMessages = ( model: unknown, body: unknown, signal: AbortSignal | undefined, - opts: MessagesUpstreamCallOptions, -) => Promise>; + opts: AnthropicMessagesUpstreamCallOptions, +) => Promise>; const candidate = (callMessages: CallMessages, upstreamId = 'up_a'): ModelCandidate => { const endpoints = { messages: {} }; @@ -80,21 +80,21 @@ const turn = (...text: readonly string[]): readonly unknown[] => const truncatedTurn = (...text: readonly string[]): readonly unknown[] => turn(...text).filter(event => event !== messageStop); -const stream = (events: readonly unknown[]): ProviderStreamResult => ({ +const stream = (events: readonly unknown[]): ProviderStreamResult => ({ ok: true, modelKey: 'claude-model-key', headers: new Headers({ 'x-request-id': 'req-1', 'content-length': '99' }), events: (async function* () { - for (const event of events) yield { type: 'event' as const, event: event as MessagesStreamEvent }; + for (const event of events) yield { type: 'event' as const, event: event as AnthropicMessagesStreamEvent }; yield { type: 'done' as const }; })(), }); -const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as MessagesPayload; +const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as AnthropicMessagesPayload; /** What affinity materialized for the candidate about to be dialled. It differs from the * payload the client sent, which is the point: carried state is rewritten per candidate. */ -let affinityPayload: MessagesPayload = payload; +let affinityPayload: AnthropicMessagesPayload = payload; /** How many times the chain asked the resolver for that payload. One stage owns the reading; * anything below it reads the record. */ @@ -108,12 +108,12 @@ const serveWith = async ( wantsStream: boolean, headers: readonly (readonly [string, string])[] = [], ) => await run( - messagesServePipeline(payload), + anthropicMessagesServePipeline(payload), move({ 'ingress.http.headers': headers, 'ingress.chat.sourceProtocol': 'messages', - 'ingress.chat.messages.wantsStream': wantsStream, - 'request.chat.messages': payload, + 'ingress.chat.anthropicMessages.wantsStream': wantsStream, + 'request.chat.anthropicMessages': payload, 'serve.model': 'claude-model', }) as never, { @@ -186,7 +186,7 @@ describe('the messages chain', () => { // the chain had touched. it('puts the payload this attempt is owed into the record, and sends that', async () => { let sent: Record | undefined; - affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as MessagesPayload; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as AnthropicMessagesPayload; resolves([candidate(async (_model, body) => { sent = body as Record; return stream(turn('hi')); @@ -197,7 +197,7 @@ describe('the messages chain', () => { // The exit contract names what the run answers with, and the request the chain sent is // not that — but the record carries every key in flight, so what was sent is still there // to read. - expect((facts as Record)['request.chat.messages']).toBe(affinityPayload); + expect((facts as Record)['request.chat.anthropicMessages']).toBe(affinityPayload); expect(asked).toBe(1); // The id the client addressed does not travel: the provider re-stamps what it resolved. expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); @@ -210,7 +210,7 @@ describe('the messages chain', () => { resolves([candidate(async () => stream(turn('he', 'llo')))]); const { facts } = await serve(true); - const frames = withoutCarrier(await collect(facts['response.chat.messages.rendered'])); + const frames = withoutCarrier(await collect(facts['response.chat.anthropicMessages.rendered'])); expect(facts['response.http.status']).toBe(200); expect(frames.map(frame => frame.event)).toEqual([ @@ -228,7 +228,7 @@ describe('the messages chain', () => { const gateway = mockChatGatewayCtx({ wantsStream: true }); const { facts } = await serveWith(gateway, true); - const frames = await collect(facts['response.chat.messages.rendered']); + const frames = await collect(facts['response.chat.anthropicMessages.rendered']); const carrier = frames .map(frame => JSON.parse(frame.data) as { readonly type: string; readonly content_block?: { readonly type: string; readonly data: string } }) .filter(event => event.type === 'content_block_start' && event.content_block?.type === 'redacted_thinking'); @@ -248,7 +248,7 @@ describe('the messages chain', () => { resolves([candidate(async () => stream(turn('he', 'llo')))]); const { facts } = await serve(false); - const rendered = facts['response.chat.messages.rendered'] as { + const rendered = facts['response.chat.anthropicMessages.rendered'] as { readonly type: string; readonly content: readonly { readonly type: string; readonly text?: string }[]; readonly stop_reason: string; @@ -285,7 +285,7 @@ describe('the messages chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(429); - expect(facts['response.chat.messages.rendered']).toEqual({ + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ type: 'error', error: { type: 'rate_limit_error', message: 'slow down' }, }); @@ -304,14 +304,14 @@ describe('the messages chain', () => { expect(tried).toEqual(['dead', 'alive']); expect(facts['response.http.status']).toBe(200); - expect(facts['response.chat.messages.rendered']).toMatchObject({ type: 'message' }); + expect(facts['response.chat.anthropicMessages.rendered']).toMatchObject({ type: 'message' }); }); // Anthropic beta flags have a typed path of their own precisely so no provider's header // allowlist can admit them and no other source protocol can leak them in — so the field is // read off the client's request and does not travel beside itself. it('hands the beta flags over on their own path and not as a header', async () => { - let seen: MessagesUpstreamCallOptions | undefined; + let seen: AnthropicMessagesUpstreamCallOptions | undefined; resolves([candidate(async (_model, _body, _signal, opts) => { seen = opts; return stream(turn('hi')); @@ -341,8 +341,8 @@ describe('the messages chain', () => { { identity: { model: 'claude-model', upstream: 'up_a', modelKey: 'claude-model-key', pricing: null }, quantities: {} }, ]); - await collect(facts['response.chat.messages.rendered']); - const outcome = await facts['response.chat.messages.streamedUsage']!; + await collect(facts['response.chat.anthropicMessages.rendered']); + const outcome = await facts['response.chat.anthropicMessages.streamedUsage']!; const billable = outcome.billable; expect(billable[0]).toMatchObject({ @@ -363,7 +363,7 @@ describe('the messages chain', () => { const { facts } = await serveWith(gateway, true); expect(gateway.attempt.firstOutputTokenAt).toBeNull(); - await collect(facts['response.chat.messages.rendered']); + await collect(facts['response.chat.anthropicMessages.rendered']); expect(gateway.attempt.firstOutputTokenAt).toBeTypeOf('number'); }); @@ -375,7 +375,7 @@ describe('the messages chain', () => { resolves([candidate(async () => stream([...turn('hi'), textDelta(' and more')]))]); const { facts } = await serve(true); - const frames = withoutCarrier(await collect(facts['response.chat.messages.rendered'])); + const frames = withoutCarrier(await collect(facts['response.chat.anthropicMessages.rendered'])); expect(frames.map(frame => frame.event)).toEqual([ 'message_start', 'content_block_start', 'content_block_delta', @@ -391,6 +391,6 @@ describe('the messages chain', () => { const { facts } = await serve(true); - await expect(collect(facts['response.chat.messages.rendered'])).rejects.toThrow(MESSAGES_MISSING_TERMINAL_MESSAGE); + await expect(collect(facts['response.chat.anthropicMessages.rendered'])).rejects.toThrow(MESSAGES_MISSING_TERMINAL_MESSAGE); }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts index 2048e79d03..dee235c746 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts @@ -11,9 +11,9 @@ import { summarizationTurnFor, summaryTextFrom, SUMMARY_PREFIX, -} from '../../../../src/data-plane/chat/responses/compact-shim.ts'; +} from '../../../../src/data-plane/chat/openai-responses/compact-shim.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../src/shared/base64url-json.ts'; -import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesOutputItem, ResponsesResult } from '@floway-dev/protocols/responses'; +import type { CanonicalResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; import { assertEquals } from '@floway-dev/test-utils'; const turnFor = (payload: Partial = {}): CanonicalResponsesPayload => @@ -24,7 +24,7 @@ const itemsOf = (payload: CanonicalResponsesPayload): { type: string; role?: str payload.input as { type: string; role?: string; content?: { type: string; text: string }[] }[]; /** What the summarization turn produced, as an upstream states it. */ -const summarized = (text: string, overrides: Partial = {}): ResponsesResult => ({ +const summarized = (text: string, overrides: Partial = {}): OpenAIResponsesResult => ({ id: 'resp_fake_upstream', object: 'response', model: 'test-upstream-model', @@ -185,7 +185,7 @@ test('a history ending on an assistant message gets a synthetic terminal user pr // ── The summary, and the envelope it is packed into ────────────────────────── test('the summary is the item the turn closed, not the output its terminal stated', () => { - const message: ResponsesOutputItem = { + const message: OpenAIResponsesOutputItem = { type: 'message', id: 'msg_1', role: 'assistant', @@ -228,7 +228,7 @@ test("the upstream's `output_text` SDK alias is dropped from the synthesized env const envelope = buildCompactionEnvelope( 'cmp_1', 'THE SUMMARY', - summarized('THE SUMMARY', { output_text: 'THE SUMMARY' } as Partial), + summarized('THE SUMMARY', { output_text: 'THE SUMMARY' } as Partial), ); assertEquals(envelope.output_text, undefined); @@ -257,7 +257,7 @@ test('round-trip: what one turn packed is what the next turn is sent', () => { // with the summary message, carrying the handoff prefix baked in at encode time. const expanded = expandShimCompactionItems({ model: 'test-model', - input: [{ type: 'compaction', id: item.id, encrypted_content: item.encrypted_content } as unknown as ResponsesInputItem], + input: [{ type: 'compaction', id: item.id, encrypted_content: item.encrypted_content } as unknown as OpenAIResponsesInputItem], }); const items = expanded.input as { type: string; role: string; content: { type: string; text: string }[] }[]; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts index 03c580fb1b..a8a388d810 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts @@ -4,12 +4,12 @@ import { test } from 'vitest'; -import type { ResponsesServeFailure } from '../../../../src/data-plane/chat/responses/errors.ts'; +import type { OpenAIResponsesServeFailure } from '../../../../src/data-plane/chat/openai-responses/errors.ts'; import { throwChatServeFailure, tryCatchChatServeFailure } from '../../../../src/data-plane/chat/shared/errors.ts'; import { assertEquals, assertThrows } from '@floway-dev/test-utils'; test('round-trips the Responses-only item-not-found failure through throw/catch', () => { - const failure: ResponsesServeFailure = { kind: 'item-not-found', itemId: 'msg_abc' }; + const failure: OpenAIResponsesServeFailure = { kind: 'item-not-found', itemId: 'msg_abc' }; const error = assertThrows(() => throwChatServeFailure(failure)); - assertEquals(tryCatchChatServeFailure(error), failure); + assertEquals(tryCatchChatServeFailure(error), failure); }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts index 56415c85e4..2c23593a76 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts @@ -565,7 +565,7 @@ test('POST /v1/responses replays a stored turn when the next one names it as pre body: JSON.stringify({ model: 'test-model', input: 'hello', store: true }), }); assertEquals(first.status, 200); - const stored = await first.json() as ResponsesResult; + const stored = await first.json() as OpenAIResponsesResult; const second = await app.request('/v1/responses', { method: 'POST', diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index 96947cb298..52a40cd22e 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -2,8 +2,8 @@ import type { ExecutionContext } from 'hono'; import { onTestFinished, test, vi } from 'vitest'; import { app } from '../../../../src/app.ts'; -import { hashResponsesItem } from '../../../../src/data-plane/chat/responses/items/identity.ts'; -import { KEEP_ALIVE_EVENT_TYPE } from '../../../../src/data-plane/chat/responses/websocket.ts'; +import { hashResponsesItem } from '../../../../src/data-plane/chat/openai-responses/items/identity.ts'; +import { KEEP_ALIVE_EVENT_TYPE } from '../../../../src/data-plane/chat/openai-responses/websocket.ts'; import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../../../src/data-plane/shared/sse.ts'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../../src/repo/usage-metrics.ts'; @@ -411,7 +411,7 @@ test('Responses WebSocket rejects the next turn after its API key is rotated', a test('Responses WebSocket reports a failed turn when an output item cannot be persisted', async () => { const { apiKey, repo } = await setupAppTest(); - const persistence = vi.spyOn(repo.responsesItems, 'insertMany').mockRejectedValue(new Error('simulated item persistence failure')); + const persistence = vi.spyOn(repo.openaiResponsesItems, 'insertMany').mockRejectedValue(new Error('simulated item persistence failure')); try { await withMockedFetch( async request => { @@ -851,15 +851,15 @@ test('Responses WebSocket store:false keeps session snapshots without durable re const firstResponseId = terminalResponseId(firstMessages); assert(firstResponseId !== 'resp_ws_store_false_1', 'expected the source boundary to replace the upstream response id'); - assertEquals(await repo.responsesSnapshots.lookup(apiKey.id, firstResponseId, 0), null); + assertEquals(await repo.openaiResponsesSnapshots.lookup(apiKey.id, firstResponseId, 0), null); const firstOutput = firstMessages.find(message => message.type === 'response.output_item.done' && (message as { item?: { type?: unknown } }).item?.type === 'message') as { item?: { id?: string } } | undefined; assertExists(firstOutput?.item?.id); assert(firstOutput.item.id !== 'assistant_ws_store_false_1', 'expected Copilot to replace the raw message id'); - assertEquals(await repo.responsesItems.lookupMany(apiKey.id, [firstOutput.item.id], 0), []); + assertEquals(await repo.openaiResponsesItems.lookupMany(apiKey.id, [firstOutput.item.id], 0), []); assertEquals( - await repo.responsesItems.lookupManyByItemHash(apiKey.id, [await hashResponsesItem({ type: 'message', role: 'user', content: 'first question' })], 0), + await repo.openaiResponsesItems.lookupManyByItemHash(apiKey.id, [await hashResponsesItem({ type: 'message', role: 'user', content: 'first question' })], 0), [], ); @@ -876,7 +876,7 @@ test('Responses WebSocket store:false keeps session snapshots without durable re })); const secondMessages = await followupTerminal; const secondResponseId = terminalResponseId(secondMessages); - assertEquals(await repo.responsesSnapshots.lookup(apiKey.id, secondResponseId, 0), null); + assertEquals(await repo.openaiResponsesSnapshots.lookup(apiKey.id, secondResponseId, 0), null); const secondBody = upstreamBodies[1] as { previous_response_id?: unknown; input: Array<{ type: string; role?: string; content?: unknown }> }; assertEquals(secondBody.previous_response_id, undefined); @@ -1172,8 +1172,8 @@ test('Responses WebSocket store:true durable snapshots can chain through local s }), ); - const firstSnapshot = await repo.responsesSnapshots.lookup(apiKey.id, firstResponseId!, 0); - const secondSnapshot = await repo.responsesSnapshots.lookup(apiKey.id, secondResponseId!, 0); + const firstSnapshot = await repo.openaiResponsesSnapshots.lookup(apiKey.id, firstResponseId!, 0); + const secondSnapshot = await repo.openaiResponsesSnapshots.lookup(apiKey.id, secondResponseId!, 0); assertExists(firstSnapshot); assertExists(secondSnapshot); assertEquals(secondSnapshot.itemIds.length > firstSnapshot.itemIds.length, true); @@ -1336,10 +1336,10 @@ test('Responses WebSocket session-level store: second message resolves prior ite // The first turn wrote to both the durable repo and the session-local // cache. Wipe the repo to prove the next lookup comes from the cache // alone. - assertExists(await repo.responsesSnapshots.lookup(apiKey.id, firstResponseId, 0)); - await repo.responsesSnapshots.deleteAll(); - await repo.responsesItems.deleteAll(); - assertEquals(await repo.responsesSnapshots.lookup(apiKey.id, firstResponseId, 0), null); + assertExists(await repo.openaiResponsesSnapshots.lookup(apiKey.id, firstResponseId, 0)); + await repo.openaiResponsesSnapshots.deleteAll(); + await repo.openaiResponsesItems.deleteAll(); + assertEquals(await repo.openaiResponsesSnapshots.lookup(apiKey.id, firstResponseId, 0), null); const secondTerminal = waitForMessages(sessionA, messages => messages.some(isTerminalResponseEvent)); sessionA.send(JSON.stringify({ @@ -1363,11 +1363,11 @@ test('Responses WebSocket session-level store: second message resolves prior ite ['message', 'user', 'turn two input'], ]); - const restored = await repo.responsesSnapshots.lookup(apiKey.id, firstResponseId, 0); + const restored = await repo.openaiResponsesSnapshots.lookup(apiKey.id, firstResponseId, 0); assertExists(restored); - assertEquals((await repo.responsesItems.lookupMany(apiKey.id, restored.itemIds, 0)).length, restored.itemIds.length); - await repo.responsesSnapshots.deleteAll(); - await repo.responsesItems.deleteAll(); + assertEquals((await repo.openaiResponsesItems.lookupMany(apiKey.id, restored.itemIds, 0)).length, restored.itemIds.length); + await repo.openaiResponsesSnapshots.deleteAll(); + await repo.openaiResponsesItems.deleteAll(); // A fresh WS session for the same api key has its own empty cache; with // the repo wiped, the snapshot is unreachable. @@ -1401,7 +1401,7 @@ test('Responses WebSocket session-level store: second message resolves prior ite test('Responses WebSocket aborts the in-flight Responses request when the client closes', async () => { const { apiKey } = await setupAppTest(); let resolveResponsesStarted: (() => void) | undefined; - const responsesStarted = new Promise(resolve => { + const openaiResponsesStarted = new Promise(resolve => { resolveResponsesStarted = resolve; }); let resolveUpstreamAborted: (() => void) | undefined; @@ -1447,7 +1447,7 @@ test('Responses WebSocket aborts the in-flight Responses request when the client }, })); - await responsesStarted; + await openaiResponsesStarted; client.close(); await upstreamAborted; }), diff --git a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts index 1735aec739..a8aebaff51 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts @@ -9,16 +9,16 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/responses/compact-shim.ts'; -import { responsesCompactPipeline } from '../../../src/data-plane/chat/responses/compact.ts'; +import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/openai-responses/compact-shim.ts'; +import { openaiResponsesCompactPipeline } from '../../../src/data-plane/chat/openai-responses/compact.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { ModelEndpoints } from '@floway-dev/protocols/common'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { CanonicalResponsesPayload, ResponsesCompactionResult, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { CanonicalResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -32,7 +32,7 @@ let live: readonly ModelCandidate[] = []; const candidate = ( calls: { callResponses?: (model: unknown, body: unknown, action: unknown) => Promise; - callMessages?: (model: unknown, body: unknown) => Promise>; + callMessages?: (model: unknown, body: unknown) => Promise>; }, overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, ): ModelCandidate => { @@ -67,7 +67,7 @@ const USAGE = { input_tokens: 12, output_tokens: 3, total_tokens: 15 }; /** What a native compaction upstream answers with: `CompactResource` states no `status`, no * `model` and no `error`. */ -const compaction = (overrides: Partial = {}): ResponsesCompactionResult => ({ +const compaction = (overrides: Partial = {}): OpenAIResponsesCompactionResult => ({ id: 'resp_upstream_compaction', object: 'response.compaction', output: [{ type: 'compaction', id: 'cmp_1', encrypted_content: 'OPAQUE_NATIVE_BLOB' } as unknown as never], @@ -77,7 +77,7 @@ const compaction = (overrides: Partial = {}): Respons const compacts = ( seen: { body?: Record; action?: unknown }, - overrides: Partial = {}, + overrides: Partial = {}, ) => async (_model: unknown, body: unknown, action: unknown): Promise => { seen.body = body as Record; seen.action = action; @@ -89,7 +89,7 @@ const generates = (text: string, seen: { body?: Record; action? async (_model: unknown, body: unknown, action: unknown): Promise => { seen.body = body as Record; seen.action = action; - const response: ResponsesResult = { + const response: OpenAIResponsesResult = { id: 'resp_summary', object: 'response', model: 'responses-model', @@ -105,7 +105,7 @@ const generates = (text: string, seen: { body?: Record; action? return { action: 'generate', ok: true, modelKey: 'responses-model-key', headers: new Headers(), events: (async function* () { - yield { type: 'event' as const, event: { type: 'response.completed', sequence_number: 0, response } as ResponsesStreamEvent }; + yield { type: 'event' as const, event: { type: 'response.completed', sequence_number: 0, response } as OpenAIResponsesStreamEvent }; yield { type: 'done' as const }; })(), }; @@ -113,8 +113,8 @@ const generates = (text: string, seen: { body?: Record; action? /** The same turn over the Messages wire, which is the only way a Messages-only candidate can * be reached: the compaction crosses into that protocol and its frames come back translated. */ -const messagesTurn = (text: string, seen: { body?: Record } = {}) => - async (_model: unknown, body: unknown): Promise> => { +const anthropicMessagesTurn = (text: string, seen: { body?: Record } = {}) => + async (_model: unknown, body: unknown): Promise> => { seen.body = body as Record; const events: unknown[] = [ { @@ -133,7 +133,7 @@ const messagesTurn = (text: string, seen: { body?: Record } = { return { ok: true, modelKey: 'responses-model-key', headers: new Headers(), events: (async function* () { - for (const event of events) yield { type: 'event' as const, event: event as MessagesStreamEvent }; + for (const event of events) yield { type: 'event' as const, event: event as AnthropicMessagesStreamEvent }; yield { type: 'done' as const }; })(), }; @@ -152,11 +152,11 @@ let gateway = mockChatGatewayCtx({ wantsStream: false }); const compact = async (request: CanonicalResponsesPayload = payload) => { gateway = mockChatGatewayCtx({ wantsStream: false }); const outcome = await run( - responsesCompactPipeline(request), + openaiResponsesCompactPipeline(request), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], 'ingress.chat.sourceProtocol': 'responses', - 'request.chat.responses': request, + 'request.chat.openaiResponses': request, 'serve.model': request.model, }) as never, { @@ -174,13 +174,13 @@ const compact = async (request: CanonicalResponsesPayload = payload) => { } as never, ); // What the epilogue would settle from, once the run has answered. - const pending = outcome.facts['response.chat.responses.streamedUsage']; + const pending = outcome.facts['response.chat.openaiResponses.streamedUsage']; if (pending !== null) settlements.push(await pending); return outcome; }; const rendered = (facts: Record): Record => - facts['response.chat.responses.rendered'] as Record; + facts['response.chat.openaiResponses.rendered'] as Record; /** The summary a simulated compaction packed into the item the client is handed. The turn's * own state is sealed around it on the way out — that is what pins the next turn to the @@ -199,8 +199,8 @@ beforeEach(() => { initRepo({ usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, - responsesSnapshots: { lookup: async () => null, put: async () => {} }, - responsesItems: { lookupMany: async () => [], putMany: async () => {} }, + openaiResponsesSnapshots: { lookup: async () => null, put: async () => {} }, + openaiResponsesItems: { lookupMany: async () => [], putMany: async () => {} }, } as never); }); @@ -244,7 +244,7 @@ describe('the responses compaction chain', () => { // wire generation would have used — which is the whole of what the action pivot was. it('simulates a compaction over a candidate with no compaction wire', async () => { const seen: { body?: Record } = {}; - resolves([candidate({ callMessages: messagesTurn('CONDENSED SUMMARY', seen) }, { endpoints: { messages: {} } })]); + resolves([candidate({ callMessages: anthropicMessagesTurn('CONDENSED SUMMARY', seen) }, { endpoints: { messages: {} } })]); const { facts } = await compact({ ...payload, @@ -371,7 +371,7 @@ describe('the responses compaction chain', () => { // the upstream actually ran — and a compaction that surfaced as failed belongs in the error // column rather than masquerading as a success. it('settles a compaction whose turn failed as a failure', async () => { - resolves([candidate({ callResponses: compacts({}, { status: 'failed' } as Partial) })]); + resolves([candidate({ callResponses: compacts({}, { status: 'failed' } as Partial) })]); await compact(); diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index eb014d02db..d291f52a3f 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -11,16 +11,16 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/responses/compact-shim.ts'; -import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; +import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/openai-responses/compact-shim.ts'; +import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { ModelEndpoints, SseFrame } from '@floway-dev/protocols/common'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import { RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type ResponsesCompactionResult, type ResponsesOutputItem, type ResponsesResult, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type OpenAIResponsesCompactionResult, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -34,7 +34,7 @@ let live: readonly ModelCandidate[] = []; const candidate = ( calls: { callResponses?: (model: unknown, body: unknown, action: unknown) => Promise; - callMessages?: (model: unknown, body: unknown) => Promise>; + callMessages?: (model: unknown, body: unknown) => Promise>; }, overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, ): ModelCandidate => { @@ -65,7 +65,7 @@ const resolves = (candidates: readonly ModelCandidate[]): void => { vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); }; -const delta = (text: string): ResponsesStreamEvent => ({ +const delta = (text: string): OpenAIResponsesStreamEvent => ({ type: 'response.output_text.delta', sequence_number: 1, item_id: 'msg_1', @@ -74,7 +74,7 @@ const delta = (text: string): ResponsesStreamEvent => ({ delta: text, }); -const result = (text: string, usage?: ResponsesResult['usage']): ResponsesResult => ({ +const result = (text: string, usage?: OpenAIResponsesResult['usage']): OpenAIResponsesResult => ({ id: 'resp_1', object: 'response', model: 'responses-model', @@ -88,7 +88,7 @@ const result = (text: string, usage?: ResponsesResult['usage']): ResponsesResult ...(usage === undefined ? {} : { usage }), }); -const completed = (text: string, usage?: ResponsesResult['usage']): ResponsesStreamEvent => ({ +const completed = (text: string, usage?: OpenAIResponsesResult['usage']): OpenAIResponsesStreamEvent => ({ type: 'response.completed', sequence_number: 2, response: result(text, usage), @@ -96,7 +96,7 @@ const completed = (text: string, usage?: ResponsesResult['usage']): ResponsesStr // The upstream's own stream, as the provider hands it up: typed frames, and the `done` frame // its SSE transport ended on. -const stream = (...events: readonly ResponsesStreamEvent[]): ProviderResponsesResult => ({ +const stream = (...events: readonly OpenAIResponsesStreamEvent[]): ProviderResponsesResult => ({ action: 'generate', ok: true, modelKey: 'responses-model-key', @@ -109,8 +109,8 @@ const stream = (...events: readonly ResponsesStreamEvent[]): ProviderResponsesRe /** One assistant message over the Messages wire, which is the only way a Messages-only * candidate can be reached: the turn crosses into that protocol and comes back translated. */ -const messagesTurn = (text: string, seen: { body?: Record } = {}) => - async (_model: unknown, body: unknown): Promise> => { +const anthropicMessagesTurn = (text: string, seen: { body?: Record } = {}) => + async (_model: unknown, body: unknown): Promise> => { seen.body = body as Record; const events: unknown[] = [ { @@ -129,7 +129,7 @@ const messagesTurn = (text: string, seen: { body?: Record } = { return { ok: true, modelKey: 'responses-model-key', headers: new Headers(), events: (async function* () { - for (const event of events) yield { type: 'event' as const, event: event as MessagesStreamEvent }; + for (const event of events) yield { type: 'event' as const, event: event as AnthropicMessagesStreamEvent }; yield { type: 'done' as const }; })(), }; @@ -156,12 +156,12 @@ const serveWith = async ( wantsStream: boolean, request: CanonicalResponsesPayload = payload, ) => await run( - responsesServePipeline(request), + openaiResponsesServePipeline(request), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], 'ingress.chat.sourceProtocol': 'responses', - 'ingress.chat.responses.wantsStream': wantsStream, - 'request.chat.responses': request, + 'ingress.chat.openaiResponses.wantsStream': wantsStream, + 'request.chat.openaiResponses': request, 'serve.model': 'responses-model', }) as never, { @@ -190,10 +190,10 @@ const drain = async (rendered: unknown): Promise => { // The carrier the edge writes back is a reasoning item holding this turn's own state. The // message this turn answers with cannot hold it, so the carrier gets an output slot of its // own at the head of the answer: two frames, and a slot in every snapshot that follows them. -const isCarrierItem = (item: ResponsesOutputItem): boolean => item.type === 'reasoning'; +const isCarrierItem = (item: OpenAIResponsesOutputItem): boolean => item.type === 'reasoning'; /** The turn's own answer, with the carrier's item taken back out. */ -const withoutCarrierItem = (response: ResponsesResult): ResponsesResult => +const withoutCarrierItem = (response: OpenAIResponsesResult): OpenAIResponsesResult => ({ ...response, output: response.output.filter(item => !isCarrierItem(item)) }); /** The turn's own frames. The two sequence numbers the carrier's frames spent stay spent — @@ -201,7 +201,7 @@ const withoutCarrierItem = (response: ResponsesResult): ResponsesResult => const withoutCarrier = (frames: readonly SseFrame[]): readonly SseFrame[] => frames.flatMap(frame => { if (frame.data === '[DONE]') return [frame]; - const event = JSON.parse(frame.data) as ResponsesStreamEvent; + const event = JSON.parse(frame.data) as OpenAIResponsesStreamEvent; if ( (event.type === 'response.output_item.added' || event.type === 'response.output_item.done') && isCarrierItem(event.item) @@ -278,7 +278,7 @@ describe('the responses chain', () => { // The exit contract names what the run answers with, and the request the chain sent is // not that — but the record carries every key in flight, so what was sent is still there // to read. - expect((facts as Record)['request.chat.responses']).toBe(affinityPayload); + expect((facts as Record)['request.chat.openaiResponses']).toBe(affinityPayload); expect(asked).toBe(1); expect(sent).toMatchObject({ input: [{ type: 'message', role: 'user', content: 'rewritten' }] }); expect(sent).not.toHaveProperty('model'); @@ -322,7 +322,7 @@ describe('the responses chain', () => { resolves([candidate({ callResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); const { facts } = await serve(true); - const frames = withoutCarrier(await drain(facts['response.chat.responses.rendered'])); + const frames = withoutCarrier(await drain(facts['response.chat.openaiResponses.rendered'])); expect(facts['response.http.status']).toBe(200); expect(frames.map(frame => frame.event)).toEqual([ @@ -340,7 +340,7 @@ describe('the responses chain', () => { // The terminal event is the upstream's, restated: the membrane at the edge stamps the // envelope id this gateway minted for the turn and completes the resource to what the // schema requires of it, so the deltas ride through untouched and this one does not. - const terminal = JSON.parse(frames[2]!.data) as { sequence_number: number; response: ResponsesResult }; + const terminal = JSON.parse(frames[2]!.data) as { sequence_number: number; response: OpenAIResponsesResult }; // The carrier's own two frames went out ahead of it under two sequence numbers of their // own, so the terminal event is numbered where they left it. expect(terminal.sequence_number).toBe(4); @@ -359,9 +359,9 @@ describe('the responses chain', () => { const gateway = mockChatGatewayCtx({ wantsStream: true }); const { facts } = await serveWith(gateway, true); - const frames = await drain(facts['response.chat.responses.rendered']); + const frames = await drain(facts['response.chat.openaiResponses.rendered']); const answered = frames.find(frame => frame.event === 'response.completed'); - const carrier = (JSON.parse(answered!.data) as { readonly response: ResponsesResult }).response.output + const carrier = (JSON.parse(answered!.data) as { readonly response: OpenAIResponsesResult }).response.output .filter(item => item.type === 'reasoning'); expect(carrier).toHaveLength(1); @@ -386,8 +386,8 @@ describe('the responses chain', () => { })]); const { facts } = await serve(true); - await drain(facts['response.chat.responses.rendered']); - const outcome = await facts['response.chat.responses.streamedUsage']!; + await drain(facts['response.chat.openaiResponses.rendered']); + const outcome = await facts['response.chat.openaiResponses.streamedUsage']!; const billable = outcome.billable; expect(billable).toEqual([expect.objectContaining({ @@ -406,7 +406,7 @@ describe('the responses chain', () => { const { facts } = await serveWith(gateway, true); expect(gateway.attempt.firstOutputTokenAt).toBeNull(); - await drain(facts['response.chat.responses.rendered']); + await drain(facts['response.chat.openaiResponses.rendered']); expect(gateway.attempt.firstOutputTokenAt).toBeTypeOf('number'); }); @@ -418,7 +418,7 @@ describe('the responses chain', () => { resolves([candidate({ callResponses: async () => stream(delta('hi'), completed('hi'), delta(' and more')) })]); const { facts } = await serve(true); - const frames = withoutCarrier(await drain(facts['response.chat.responses.rendered'])); + const frames = withoutCarrier(await drain(facts['response.chat.openaiResponses.rendered'])); expect(frames.map(frame => frame.event)).toEqual([ 'response.output_text.delta', @@ -437,7 +437,7 @@ describe('the responses chain', () => { resolves([candidate({ callResponses: async () => stream(delta('hi')) })]); const { facts } = await serve(true); - const frames = await drain(facts['response.chat.responses.rendered']); + const frames = await drain(facts['response.chat.openaiResponses.rendered']); expect(frames.map(frame => frame.event)).toEqual(['response.output_text.delta', 'error']); expect(JSON.parse(frames[1]!.data)).toMatchObject({ @@ -446,7 +446,7 @@ describe('the responses chain', () => { }); // The client was answered, and the run still failed: what the frames said on the way out // is not what the row says the turn was. - expect((await facts['response.chat.responses.streamedUsage']!).failed).toBe(true); + expect((await facts['response.chat.openaiResponses.streamedUsage']!).failed).toBe(true); }); // The upstream speaks SSE whatever the client asked for, so a client that did not ask to @@ -457,7 +457,7 @@ describe('the responses chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(200); - const answered = withoutCarrierItem(facts['response.chat.responses.rendered'] as unknown as ResponsesResult); + const answered = withoutCarrierItem(facts['response.chat.openaiResponses.rendered'] as unknown as OpenAIResponsesResult); expect(answered).toMatchObject({ status: 'completed', output: [{ content: [{ type: 'output_text', text: 'hello' }] }], @@ -470,7 +470,7 @@ describe('the responses chain', () => { // A compaction is one envelope rather than a stream: the upstream ran the turn, charged for // it, and stated the counts in the body. It rides at the same key the frames would have. it('serves an upstream that answered with one envelope as that envelope', async () => { - const compaction: ResponsesCompactionResult = { + const compaction: OpenAIResponsesCompactionResult = { id: 'resp_compact_1', object: 'response.compaction', output: [{ @@ -488,8 +488,8 @@ describe('the responses chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(200); - expect(facts['response.chat.responses.rendered']).toEqual(compaction); - expect(facts['response.chat.responses.streamedUsage']).toBeNull(); + expect(facts['response.chat.openaiResponses.rendered']).toEqual(compaction); + expect(facts['response.chat.openaiResponses.streamedUsage']).toBeNull(); expect(facts['response.usage.billable']).toEqual([expect.objectContaining({ quantities: { input_tokens: '900', output_tokens: '40' }, })]); @@ -521,7 +521,7 @@ describe('the responses chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(429); - expect(facts['response.chat.responses.rendered']).toEqual({ error: { message: 'slow down' } }); + expect(facts['response.chat.openaiResponses.rendered']).toEqual({ error: { message: 'slow down' } }); }); // A refused connection is an outcome the fork has to be able to see, not a fault that ends @@ -585,7 +585,7 @@ describe('the responses chain', () => { expect(sent).toMatchObject({ store: false }); expect(facts['response.http.status']).toBe(200); - const answered = facts['response.chat.responses.rendered'] as Record; + const answered = facts['response.chat.openaiResponses.rendered'] as Record; expect(answered.object).toBe('response.compaction'); expect(await summaryIn(gateway, compactionItem(answered).encrypted_content)).toBe(`${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); }); @@ -609,7 +609,7 @@ describe('the responses chain', () => { expect(sent).toMatchObject({ input: [{ type: 'message', role: 'user', content: 'hi' }] }); expect(JSON.stringify(sent)).not.toContain('CONTEXT CHECKPOINT COMPACTION'); - const answered = withoutCarrierItem(facts['response.chat.responses.rendered'] as unknown as ResponsesResult); + const answered = withoutCarrierItem(facts['response.chat.openaiResponses.rendered'] as unknown as OpenAIResponsesResult); expect(answered.object).toBe('response'); expect(answered).toMatchObject({ output: [{ content: [{ type: 'output_text', text: 'hello' }] }] }); }); @@ -619,7 +619,7 @@ describe('the responses chain', () => { // the upstream's own — its id, and a blob only it can read. it('sends a turn that asked for a compaction on unchanged, where the shim is not engaged', async () => { let sent: Record | undefined; - const upstreamCompaction: ResponsesCompactionResult = { + const upstreamCompaction: OpenAIResponsesCompactionResult = { id: 'resp_upstream_compaction', object: 'response.compaction', output: [{ type: 'compaction', id: 'cmp_1', encrypted_content: 'OPAQUE_NATIVE_BLOB' } as unknown as never], @@ -637,7 +637,7 @@ describe('the responses chain', () => { expect(sent).toMatchObject({ input: [{ role: 'user' }, { type: 'compaction_trigger' }] }); expect(JSON.stringify(sent)).not.toContain('CONTEXT CHECKPOINT COMPACTION'); - expect(facts['response.chat.responses.rendered']).toEqual(upstreamCompaction); + expect(facts['response.chat.openaiResponses.rendered']).toEqual(upstreamCompaction); }); // No translation carries a compaction, and neither translator models the item that asks for @@ -647,7 +647,7 @@ describe('the responses chain', () => { const seen: { body?: Record } = {}; affinityPayload = asksForCompaction; resolves([candidate( - { callMessages: messagesTurn('CONDENSED SUMMARY', seen) }, + { callMessages: anthropicMessagesTurn('CONDENSED SUMMARY', seen) }, { endpoints: { messages: {} } }, )]); const gateway = mockChatGatewayCtx({ wantsStream: false }); @@ -656,7 +656,7 @@ describe('the responses chain', () => { expect(JSON.stringify(seen.body)).toContain('CONTEXT CHECKPOINT COMPACTION'); expect(JSON.stringify(seen.body)).not.toContain('compaction_trigger'); - const answered = facts['response.chat.responses.rendered'] as Record; + const answered = facts['response.chat.openaiResponses.rendered'] as Record; expect(answered.object).toBe('response.compaction'); expect(await summaryIn(gateway, compactionItem(answered).encrypted_content)).toBe(`${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); }); diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts index 2940820a08..ce88506579 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts @@ -40,7 +40,7 @@ const withCopilot = async ( /** The answer a Chat Completions upstream gives, as the SSE every chat endpoint really speaks. * Gemini's first-preference wire, so this is what most of these turns are dialled on. */ -const chatCompletionsTurn = (): Response => +const openaiChatCompletionsTurn = (): Response => sseChatCompletionsResponse({ id: 'chatcmpl_route', object: 'chat.completion', @@ -57,14 +57,14 @@ const post = async (apiKey: string, modelAction: string): Promise => body: JSON.stringify({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }), }); -interface GeminiEvent { +interface GeminiGenerateContentEvent { readonly candidates?: readonly { readonly content: { readonly parts: readonly { readonly text?: string }[] }; readonly finishReason?: string; }[]; } -const geminiText = (event: GeminiEvent): string => +const geminiGenerateContentText = (event: GeminiGenerateContentEvent): string => (event.candidates ?? []).flatMap(candidate => candidate.content.parts.map(part => part.text ?? '')).join(''); test('a streaming turn reaches the client as Gemini SSE, ending on the frame that states a finish reason', async () => { @@ -78,7 +78,7 @@ test('a streaming turn reaches the client as Gemini SSE, ending on the frame tha // Chat Completions body, streaming because every chat endpoint here does. expect(body.messages).toBeDefined(); expect(body.stream).toBe(true); - return chatCompletionsTurn(); + return openaiChatCompletionsTurn(); }, async () => await post(apiKey.key, 'gpt-route:streamGenerateContent'), ); @@ -90,9 +90,9 @@ test('a streaming turn reaches the client as Gemini SSE, ending on the frame tha expect(frames.length > 0).toBe(true); // One JSON object per `data:` line, and no sentinel — Gemini has none to write, so a `[DONE]` // here would be a frame no Google client can parse. - const events = frames.map(frame => JSON.parse(frame.data) as GeminiEvent); + const events = frames.map(frame => JSON.parse(frame.data) as GeminiGenerateContentEvent); for (const frame of frames) expect(frame.event).toBe('message'); - expect(events.map(geminiText).join('')).toBe(ANSWER); + expect(events.map(geminiGenerateContentText).join('')).toBe(ANSWER); // The turn's end is stated on the frames the client is handed, not on the wire below them. expect(events.at(-1)?.candidates?.[0]?.finishReason).toBe('STOP'); expect(events.slice(0, -1).every(event => event.candidates?.[0]?.finishReason === undefined)).toBe(true); @@ -103,17 +103,17 @@ test('a non-streaming turn reaches the client as one assembled Gemini body', asy const response = await withCopilot( [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], - () => chatCompletionsTurn(), + () => openaiChatCompletionsTurn(), async () => await post(apiKey.key, 'gpt-route:generateContent'), ); expect(response.status).toBe(200); expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); - const body = await response.json() as GeminiEvent & { usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number } }; + const body = await response.json() as GeminiGenerateContentEvent & { usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number } }; expect(body.candidates).toHaveLength(1); expect(body.candidates?.[0].finishReason).toBe('STOP'); - expect(geminiText(body)).toBe(ANSWER); + expect(geminiGenerateContentText(body)).toBe(ANSWER); // What the upstream metered is stated to the client too, in this protocol's own names. expect(body.usageMetadata?.promptTokenCount).toBe(21); expect(body.usageMetadata?.candidatesTokenCount).toBe(3); @@ -171,7 +171,7 @@ test('a served turn writes one usage row carrying the tokens the upstream report const response = await withCopilot( [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], - () => chatCompletionsTurn(), + () => openaiChatCompletionsTurn(), async () => { const answer = await post(apiKey.key, 'gpt-route:streamGenerateContent'); // A stream's numbers arrive with its last chunk, so the row is written after the client @@ -220,10 +220,10 @@ test('a turn dialled over the Messages wire still answers the client in Gemini', ); expect(response.status).toBe(200); - const body = await response.json() as GeminiEvent; + const body = await response.json() as GeminiGenerateContentEvent; expect(body.candidates).toHaveLength(1); expect(body.candidates?.[0].finishReason).toBe('STOP'); - expect(geminiText(body)).toBe(ANSWER); + expect(geminiGenerateContentText(body)).toBe(ANSWER); await flushBackground(); const rows = await repo.usage.listAll(); diff --git a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts index 267d3cd206..73c4f95ee7 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts @@ -58,7 +58,7 @@ const post = async (apiKey: string, body: Record): Promise = {}): Record => ({ model, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }], ...extra }); -interface MessagesBody { +interface AnthropicMessagesBody { readonly type: string; readonly role: string; readonly stop_reason: string; @@ -67,7 +67,7 @@ interface MessagesBody { } /** The answer's own text, past the carrier the edge writes back. */ -const textOf = (body: MessagesBody): string | undefined => +const textOf = (body: AnthropicMessagesBody): string | undefined => body.content.find(block => block.type === 'text')?.text; test('a streaming turn reaches the client as Messages SSE, message_stop included', async () => { @@ -118,7 +118,7 @@ test('a non-streaming turn reaches the client as one assembled Messages body', a expect(response.status).toBe(200); expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); - const body = await response.json() as MessagesBody; + const body = await response.json() as AnthropicMessagesBody; expect(body.type).toBe('message'); expect(body.role).toBe('assistant'); expect(body.stop_reason).toBe('end_turn'); @@ -216,7 +216,7 @@ test('a turn dialled over the Chat Completions wire still answers the client in ); expect(response.status).toBe(200); - const body = await response.json() as MessagesBody; + const body = await response.json() as AnthropicMessagesBody; expect(body.type).toBe('message'); expect(body.role).toBe('assistant'); expect(body.stop_reason).toBe('end_turn'); diff --git a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts index 7734bedea3..92b883468f 100644 --- a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts @@ -99,7 +99,7 @@ const closedItemTypes = (frames: readonly unknown[]): readonly string[] => const ANSWER = 'hello there'; -const chatCompletionsTurn = (stream: boolean) => async (apiKey: string): Promise => +const openaiChatCompletionsTurn = (stream: boolean) => async (apiKey: string): Promise => await withCopilot( [{ id: 'gpt-frames', supported_endpoints: ['/chat/completions'] }], () => sseChatCompletionsResponse({ @@ -116,7 +116,7 @@ const chatCompletionsTurn = (stream: boolean) => async (apiKey: string): Promise }), ); -const responsesTurn = (stream: boolean) => async (apiKey: string): Promise => +const openaiResponsesTurn = (stream: boolean) => async (apiKey: string): Promise => await withCopilot( [{ id: 'gpt-responses-frames', supported_endpoints: ['/responses'] }], () => sseResponsesResponse({ @@ -173,21 +173,21 @@ const compactionTurn = async (apiKey: string): Promise => ); test('a streamed chat turn records the frames it served', async () => { - assertFramesCarry(await framesRecordedBy(chatCompletionsTurn(true)), ANSWER); + assertFramesCarry(await framesRecordedBy(openaiChatCompletionsTurn(true)), ANSWER); }); test('a collected chat turn records the frames it was assembled from', async () => { // 「流式请求前端看到流+前端组装结果,非流式请求前端看到流+原始组装结果」 — the frames are // recorded either way, which is what lets the operator see the stream behind a single body. - assertFramesCarry(await framesRecordedBy(chatCompletionsTurn(false)), ANSWER); + assertFramesCarry(await framesRecordedBy(openaiChatCompletionsTurn(false)), ANSWER); }); test('a streamed responses turn records the frames it served', async () => { - assertFramesCarry(await framesRecordedBy(responsesTurn(true)), ANSWER); + assertFramesCarry(await framesRecordedBy(openaiResponsesTurn(true)), ANSWER); }); test('a collected responses turn records the frames it was assembled from', async () => { - assertFramesCarry(await framesRecordedBy(responsesTurn(false)), ANSWER); + assertFramesCarry(await framesRecordedBy(openaiResponsesTurn(false)), ANSWER); }); test('a compaction records the frames its resource was assembled from', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 919c37b23c..160080206a 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -14,20 +14,20 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; -import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; +import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; +import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; import { handOff } from '../../../src/data-plane/chat/handoff.ts'; -import { messagesServePipeline, messagesWire } from '../../../src/data-plane/chat/messages/pipeline.ts'; -import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; +import { anthropicMessagesServePipeline, anthropicMessagesWire } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; +import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { compose, move, run, type Pipeline } from '@floway-dev/pipeline'; -import type { ChatCompletionsPayload, ChatCompletionsResult, ChatCompletionsStreamEvent } from '@floway-dev/protocols/chat-completions'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsResult, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocols/common'; -import type { GeminiPayload, GeminiResult } from '@floway-dev/protocols/gemini'; -import { PROMPT_TOO_LONG_MESSAGE, type MessagesPayload, type MessagesResult, type MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { CanonicalResponsesPayload, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { GeminiGenerateContentPayload, GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; +import { PROMPT_TOO_LONG_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesResult, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { CanonicalResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -46,8 +46,8 @@ const resolves = (candidates: readonly ModelCandidate[]): void => { }; interface Calls { - readonly callChatCompletions?: (model: unknown, body: unknown) => Promise>; - readonly callMessages?: (model: unknown, body: unknown) => Promise>; + readonly callChatCompletions?: (model: unknown, body: unknown) => Promise>; + readonly callMessages?: (model: unknown, body: unknown) => Promise>; readonly callResponses?: (model: unknown, body: unknown) => Promise; } @@ -78,7 +78,7 @@ const candidate = ( // ── What each wire answers with ─────────────────────────────────────────────────────────── -const chatCompletionsTurn = (text: string): ProviderStreamResult => ({ +const openaiChatCompletionsTurn = (text: string): ProviderStreamResult => ({ ok: true, modelKey: 'k', headers: new Headers(), @@ -86,16 +86,16 @@ const chatCompletionsTurn = (text: string): ProviderStreamResult => ({ +const anthropicMessagesTurn = (text: string): ProviderStreamResult => ({ ok: true, modelKey: 'k', headers: new Headers(), @@ -106,17 +106,17 @@ const messagesTurn = (text: string): ProviderStreamResult = id: 'msg_1', type: 'message', role: 'assistant', content: [], model: MODEL, stop_reason: null, stop_sequence: null, usage: { input_tokens: 3, output_tokens: 0 }, }, - } as MessagesStreamEvent); - yield eventFrame({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } } as MessagesStreamEvent); - yield eventFrame({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } } as MessagesStreamEvent); - yield eventFrame({ type: 'content_block_stop', index: 0 } as MessagesStreamEvent); - yield eventFrame({ type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 2 } } as MessagesStreamEvent); - yield eventFrame({ type: 'message_stop' } as MessagesStreamEvent); + } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'content_block_stop', index: 0 } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 2 } } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'message_stop' } as AnthropicMessagesStreamEvent); yield doneFrame(); })(), }); -const responsesResult = (text: string): ResponsesResult => ({ +const openaiResponsesResult = (text: string): OpenAIResponsesResult => ({ id: 'resp_1', object: 'response', model: MODEL, @@ -129,7 +129,7 @@ const responsesResult = (text: string): ResponsesResult => ({ incomplete_details: null, }); -const responsesTurn = (text: string): ProviderResponsesResult => ({ +const openaiResponsesTurn = (text: string): ProviderResponsesResult => ({ action: 'generate', ok: true, modelKey: 'k', @@ -140,18 +140,18 @@ const responsesTurn = (text: string): ProviderResponsesResult => ({ sequence_number: 1, output_index: 0, item: { type: 'message', id: 'msg_1', role: 'assistant', status: 'in_progress', content: [] }, - } as ResponsesStreamEvent); + } as OpenAIResponsesStreamEvent); yield eventFrame({ type: 'response.output_text.delta', sequence_number: 2, item_id: 'msg_1', output_index: 0, content_index: 0, delta: text, - } as ResponsesStreamEvent); - yield eventFrame({ type: 'response.completed', sequence_number: 3, response: responsesResult(text) } as ResponsesStreamEvent); + } as OpenAIResponsesStreamEvent); + yield eventFrame({ type: 'response.completed', sequence_number: 3, response: openaiResponsesResult(text) } as OpenAIResponsesStreamEvent); yield doneFrame(); })(), }); /** A refusal, in the words an OpenAI-shaped upstream uses for a turn that will not fit. */ -const contextExceeded = async (): Promise> => ({ +const contextExceeded = async (): Promise> => ({ ok: false, modelKey: 'k', response: Response.json( @@ -183,68 +183,68 @@ const serve = async ( } as never); }; -const chatCompletionsPayload = { model: MODEL, messages: [{ role: 'user', content: 'hi' }] } as unknown as ChatCompletionsPayload; -const messagesPayload = { model: MODEL, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as MessagesPayload; -const responsesPayload = { model: MODEL, input: [{ type: 'message', role: 'user', content: 'hi' }] } as unknown as CanonicalResponsesPayload; -const geminiPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiPayload; +const openaiChatCompletionsPayload = { model: MODEL, messages: [{ role: 'user', content: 'hi' }] } as unknown as OpenAIChatCompletionsPayload; +const anthropicMessagesPayload = { model: MODEL, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as AnthropicMessagesPayload; +const openaiResponsesPayload = { model: MODEL, input: [{ type: 'message', role: 'user', content: 'hi' }] } as unknown as CanonicalResponsesPayload; +const geminiGenerateContentPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiGenerateContentPayload; -const serveChatCompletions = async (payload: ChatCompletionsPayload = chatCompletionsPayload) => - await serve(chatCompletionsServePipeline(payload), { +const serveChatCompletions = async (payload: OpenAIChatCompletionsPayload = openaiChatCompletionsPayload) => + await serve(openaiChatCompletionsServePipeline(payload), { 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'chatCompletions', - 'ingress.chat.chatCompletions.wantsStream': false, - 'ingress.chat.chatCompletions.wantsUsageChunk': false, - 'request.chat.chatCompletions': payload, + 'ingress.chat.openaiChatCompletions.wantsStream': false, + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false, + 'request.chat.openaiChatCompletions': payload, 'serve.model': MODEL, }, payload); -const serveMessages = async (payload: MessagesPayload = messagesPayload) => - await serve(messagesServePipeline(payload), { +const serveMessages = async (payload: AnthropicMessagesPayload = anthropicMessagesPayload) => + await serve(anthropicMessagesServePipeline(payload), { 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'messages', - 'ingress.chat.messages.wantsStream': false, - 'request.chat.messages': payload, + 'ingress.chat.anthropicMessages.wantsStream': false, + 'request.chat.anthropicMessages': payload, 'serve.model': MODEL, }, payload); -const serveResponses = async (payload: CanonicalResponsesPayload = responsesPayload) => - await serve(responsesServePipeline(payload), { +const serveResponses = async (payload: CanonicalResponsesPayload = openaiResponsesPayload) => + await serve(openaiResponsesServePipeline(payload), { 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'responses', - 'ingress.chat.responses.wantsStream': false, - 'request.chat.responses': payload, + 'ingress.chat.openaiResponses.wantsStream': false, + 'request.chat.openaiResponses': payload, 'serve.model': MODEL, }, payload); -const serveGemini = async (payload: GeminiPayload = geminiPayload) => - await serve(geminiServePipeline(payload), { +const serveGemini = async (payload: GeminiGenerateContentPayload = geminiGenerateContentPayload) => + await serve(geminiGenerateContentServePipeline(payload), { 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'gemini', - 'ingress.chat.gemini.wantsStream': false, - 'request.chat.gemini': payload, + 'ingress.chat.geminiGenerateContent.wantsStream': false, + 'request.chat.geminiGenerateContent': payload, 'serve.model': MODEL, }, payload); // ── What the client was answered with, per protocol ─────────────────────────────────────── -const chatCompletionsText = (rendered: unknown): string | null | undefined => - (rendered as ChatCompletionsResult).choices[0]?.message.content as string | null | undefined; +const openaiChatCompletionsText = (rendered: unknown): string | null | undefined => + (rendered as OpenAIChatCompletionsResult).choices[0]?.message.content as string | null | undefined; /** The answer's own text, past the carrier the edge writes back: on this protocol the turn's * state rides as a redacted-thinking block at the head of the content. */ -const messagesText = (rendered: unknown): string | undefined => { - const block = (rendered as MessagesResult).content.find(part => part.type === 'text'); +const anthropicMessagesText = (rendered: unknown): string | undefined => { + const block = (rendered as AnthropicMessagesResult).content.find(part => part.type === 'text'); return block?.type === 'text' ? block.text : undefined; }; -const responsesText = (rendered: unknown): string | undefined => { - const item = (rendered as ResponsesResult).output.find(output => output.type === 'message'); +const openaiResponsesText = (rendered: unknown): string | undefined => { + const item = (rendered as OpenAIResponsesResult).output.find(output => output.type === 'message'); const part = item?.type === 'message' ? item.content[0] : undefined; return part?.type === 'output_text' ? part.text : undefined; }; -const geminiText = (rendered: unknown): string | undefined => - (rendered as GeminiResult).candidates?.[0]?.content.parts.map(part => part.text ?? '').join(''); +const geminiGenerateContentText = (rendered: unknown): string | undefined => + (rendered as GeminiGenerateContentResult).candidates?.[0]?.content.parts.map(part => part.text ?? '').join(''); beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); @@ -256,103 +256,103 @@ beforeEach(() => { describe('a chat family reaching a candidate over another protocol', () => { it('serves /v1/chat/completions on a Messages-only candidate', async () => { - const callMessages = vi.fn(async () => messagesTurn('hello')); + const callMessages = vi.fn(async () => anthropicMessagesTurn('hello')); resolves([candidate('up_a', { messages: {} }, { callMessages })]); const { facts, drain } = await serveChatCompletions(); expect(callMessages).toHaveBeenCalledTimes(1); expect(facts['response.http.status']).toBe(200); - expect(chatCompletionsText(facts['response.chat.chatCompletions.rendered'])).toBe('hello'); + expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/chat/completions on a Responses-only candidate', async () => { - const callResponses = vi.fn(async () => responsesTurn('hello')); + const callResponses = vi.fn(async () => openaiResponsesTurn('hello')); resolves([candidate('up_a', { responses: {} }, { callResponses })]); const { facts, drain } = await serveChatCompletions(); expect(callResponses).toHaveBeenCalledTimes(1); - expect(chatCompletionsText(facts['response.chat.chatCompletions.rendered'])).toBe('hello'); + expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/messages on a Responses-only candidate', async () => { - const callResponses = vi.fn(async () => responsesTurn('hello')); + const callResponses = vi.fn(async () => openaiResponsesTurn('hello')); resolves([candidate('up_a', { responses: {} }, { callResponses })]); const { facts, drain } = await serveMessages(); expect(callResponses).toHaveBeenCalledTimes(1); - expect(messagesText(facts['response.chat.messages.rendered'])).toBe('hello'); + expect(anthropicMessagesText(facts['response.chat.anthropicMessages.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/messages on a Chat Completions-only candidate', async () => { - const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); + const callChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); const { facts, drain } = await serveMessages(); expect(callChatCompletions).toHaveBeenCalledTimes(1); - expect(messagesText(facts['response.chat.messages.rendered'])).toBe('hello'); + expect(anthropicMessagesText(facts['response.chat.anthropicMessages.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/responses on a Messages-only candidate', async () => { - const callMessages = vi.fn(async () => messagesTurn('hello')); + const callMessages = vi.fn(async () => anthropicMessagesTurn('hello')); resolves([candidate('up_a', { messages: {} }, { callMessages })]); const { facts, drain } = await serveResponses(); expect(callMessages).toHaveBeenCalledTimes(1); - expect(responsesText(facts['response.chat.responses.rendered'])).toBe('hello'); + expect(openaiResponsesText(facts['response.chat.openaiResponses.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/responses on a Chat Completions-only candidate', async () => { - const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); + const callChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); const { facts, drain } = await serveResponses(); expect(callChatCompletions).toHaveBeenCalledTimes(1); - expect(responsesText(facts['response.chat.responses.rendered'])).toBe('hello'); + expect(openaiResponsesText(facts['response.chat.openaiResponses.rendered'])).toBe('hello'); await drain(); }); // Gemini has no wire of its own, so all three of its rows are translated ones. it('serves :generateContent on a Chat Completions-only candidate', async () => { - const callChatCompletions = vi.fn(async () => chatCompletionsTurn('hello')); + const callChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); const { facts, drain } = await serveGemini(); expect(callChatCompletions).toHaveBeenCalledTimes(1); - expect(geminiText(facts['response.chat.gemini.rendered'])).toBe('hello'); + expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); await drain(); }); it('serves :generateContent on a Messages-only candidate', async () => { - const callMessages = vi.fn(async () => messagesTurn('hello')); + const callMessages = vi.fn(async () => anthropicMessagesTurn('hello')); resolves([candidate('up_a', { messages: {} }, { callMessages })]); const { facts, drain } = await serveGemini(); expect(callMessages).toHaveBeenCalledTimes(1); - expect(geminiText(facts['response.chat.gemini.rendered'])).toBe('hello'); + expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); await drain(); }); it('serves :generateContent on a Responses-only candidate', async () => { - const callResponses = vi.fn(async () => responsesTurn('hello')); + const callResponses = vi.fn(async () => openaiResponsesTurn('hello')); resolves([candidate('up_a', { responses: {} }, { callResponses })]); const { facts, drain } = await serveGemini(); expect(callResponses).toHaveBeenCalledTimes(1); - expect(geminiText(facts['response.chat.gemini.rendered'])).toBe('hello'); + expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); await drain(); }); }); @@ -366,7 +366,7 @@ describe('the fork over wires', () => { const callChatCompletions = vi.fn(async () => ({ ok: false as const, modelKey: 'k', response: Response.json({ error: { message: 'slow down' } }, { status: 429 }), })); - const callMessages = vi.fn(async () => messagesTurn('second')); + const callMessages = vi.fn(async () => anthropicMessagesTurn('second')); resolves([ candidate('up_a', { chatCompletions: {} }, { callChatCompletions }), candidate('up_b', { messages: {} }, { callMessages }), @@ -377,7 +377,7 @@ describe('the fork over wires', () => { expect(callChatCompletions).toHaveBeenCalledTimes(1); expect(callMessages).toHaveBeenCalledTimes(1); expect(facts['response.http.status']).toBe(200); - expect(chatCompletionsText(facts['response.chat.chatCompletions.rendered'])).toBe('second'); + expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('second'); await drain(); }); @@ -390,7 +390,7 @@ describe('the fork over wires', () => { const { facts, drain } = await serveMessages(); expect(facts['response.http.status']).toBe(400); - expect(facts['response.chat.messages.rendered']).toEqual({ + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ type: 'error', error: { type: 'invalid_request_error', message: PROMPT_TOO_LONG_MESSAGE }, }); @@ -411,7 +411,7 @@ describe('the fork over wires', () => { const { facts, drain } = await serveMessages(); expect(facts['response.http.status']).toBe(403); - const rendered = facts['response.chat.messages.rendered'] as { type: string; error: { type: string; message: string } }; + const rendered = facts['response.chat.anthropicMessages.rendered'] as { type: string; error: { type: string; message: string } }; expect(rendered.type).toBe('error'); expect(rendered.error.type).toBe('permission_error'); // The upstream's own words survive as the sentence, which is what the replaced surface did @@ -427,7 +427,7 @@ describe('the fork over wires', () => { const { facts, drain } = await serveMessages(); - expect(facts['response.chat.messages.rendered']).toEqual({ + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ type: 'error', error: { type: 'invalid_request_error', message: PROMPT_TOO_LONG_MESSAGE }, }); @@ -447,14 +447,14 @@ describe('a rule that speaks about one protocol-s wire', () => { model: MODEL, max_tokens: 64, messages: [{ role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }], - } as unknown as MessagesPayload; + } as unknown as AnthropicMessagesPayload; resolves([candidate( 'up_a', { chatCompletions: {} }, { callChatCompletions: async (_model, body) => { sent = body as { messages: { role: string }[] }; - return chatCompletionsTurn('hello'); + return openaiChatCompletionsTurn('hello'); }, }, ['rewrite-mid-conv-system-to-user'], @@ -475,7 +475,7 @@ describe('a rule that speaks about one protocol-s wire', () => { resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions: async (_model, body) => { sent = body as { stream_options?: unknown }; - return chatCompletionsTurn('hello'); + return openaiChatCompletionsTurn('hello'); }, })]); @@ -493,17 +493,17 @@ describe('a rule that speaks about one protocol-s wire', () => { resolves([candidate('up_a', { messages: {} }, { callMessages: async (_model, body) => { sent = body as { system?: unknown }; - return messagesTurn('hello'); + return anthropicMessagesTurn('hello'); }, }, ['strip-billing-attribution'])]); const { drain } = await serveChatCompletions({ - ...chatCompletionsPayload, + ...openaiChatCompletionsPayload, messages: [ { role: 'system', content: 'be brief\nx-anthropic-billing-header: acct-42' }, { role: 'user', content: 'hi' }, ], - } as ChatCompletionsPayload); + } as OpenAIChatCompletionsPayload); expect(String(JSON.stringify(sent?.system))).not.toContain('x-anthropic-billing-header'); await drain(); @@ -518,7 +518,7 @@ describe('a rule that speaks about one protocol-s wire', () => { resolves([candidate('up_a', { messages: {} }, { callMessages: async (_model, body) => { sent = body as Record; - return messagesTurn('hello'); + return anthropicMessagesTurn('hello'); }, })]); @@ -536,18 +536,18 @@ describe('a rule that speaks about one protocol-s wire', () => { // translated turn is that the rule lives in the wire's own chain. it('cannot be placed in a chain whose payload a handoff consumed', () => { const leaving = handOff({ - from: { request: 'request.chat.messages', response: 'response.chat.messages' }, - to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + from: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, trip: () => { throw new Error('not dialled'); }, }); const arriving = handOff({ - from: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, - to: { request: 'request.chat.messages', response: 'response.chat.messages' }, + from: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, trip: () => { throw new Error('not dialled'); }, }); - expect(() => compose('leaving', [leaving, ...messagesWire('r')])) + expect(() => compose('leaving', [leaving, ...anthropicMessagesWire('r')])) .toThrow(/needs request\.chat\.messages, which handOff:.* consumed above it/); - expect(() => compose('arriving', [arriving, ...messagesWire('r')])).not.toThrow(); + expect(() => compose('arriving', [arriving, ...anthropicMessagesWire('r')])).not.toThrow(); }); }); diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 97fabe82bb..1486b008bd 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -8,13 +8,13 @@ import { describe, expect, it } from 'vitest'; import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/pipeline.ts'; -import { chatCompletionsServePipeline } from '../../../src/data-plane/chat/chat-completions/pipeline.ts'; -import { geminiCountTokensPipeline } from '../../../src/data-plane/chat/gemini/count-tokens.ts'; -import { geminiServePipeline } from '../../../src/data-plane/chat/gemini/pipeline.ts'; -import { messagesCountTokensPipeline } from '../../../src/data-plane/chat/messages/count-tokens.ts'; -import { messagesServePipeline } from '../../../src/data-plane/chat/messages/pipeline.ts'; -import { responsesCompactPipeline } from '../../../src/data-plane/chat/responses/compact.ts'; -import { responsesServePipeline } from '../../../src/data-plane/chat/responses/pipeline.ts'; +import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; +import { geminiGenerateContentCountTokensPipeline } from '../../../src/data-plane/chat/gemini-generate-content/count-tokens.ts'; +import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; +import { anthropicMessagesCountTokensPipeline } from '../../../src/data-plane/chat/anthropic-messages/count-tokens.ts'; +import { anthropicMessagesServePipeline } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; +import { openaiResponsesCompactPipeline } from '../../../src/data-plane/chat/openai-responses/compact.ts'; +import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; import { imagesServePipeline } from '../../../src/data-plane/images/pipeline.ts'; @@ -28,13 +28,13 @@ const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] ['completions', () => completionsServePipeline], ['audio transcription', () => audioTranscriptionServePipeline], ['alpha search', () => searchServePipeline({ kind: 'search' } as never)], - ['chat completions', () => chatCompletionsServePipeline({ model: 'm', messages: [] } as never)], - ['messages', () => messagesServePipeline({ model: 'm', messages: [] } as never)], - ['gemini', () => geminiServePipeline({ model: 'm', contents: [] } as never)], - ['responses', () => responsesServePipeline({ model: 'm', input: [] } as never)], - ['messages count_tokens', () => messagesCountTokensPipeline({ model: 'm', messages: [] } as never)], - ['gemini countTokens', () => geminiCountTokensPipeline({ model: 'm', contents: [] } as never)], - ['responses compact', () => responsesCompactPipeline({ model: 'm', input: [] } as never)], + ['chat completions', () => openaiChatCompletionsServePipeline({ model: 'm', messages: [] } as never)], + ['messages', () => anthropicMessagesServePipeline({ model: 'm', messages: [] } as never)], + ['gemini', () => geminiGenerateContentServePipeline({ model: 'm', contents: [] } as never)], + ['responses', () => openaiResponsesServePipeline({ model: 'm', input: [] } as never)], + ['messages count_tokens', () => anthropicMessagesCountTokensPipeline({ model: 'm', messages: [] } as never)], + ['gemini countTokens', () => geminiGenerateContentCountTokensPipeline({ model: 'm', contents: [] } as never)], + ['responses compact', () => openaiResponsesCompactPipeline({ model: 'm', input: [] } as never)], ]; describe('every family assembles', () => { diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index 8af8278d07..5a3a60a9e7 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -9,7 +9,7 @@ // callMessagesCountTokensUpstream the ending: measures, and never opens a stream // // A second **operation** over this protocol rather than another wire under -// `messagesServePipeline`, which is why it is a chain of its own rather than a branch inside +// `anthropicMessagesServePipeline`, which is why it is a chain of its own rather than a branch inside // that one. It answers with a body and nothing else: no stream, and no billed turn — the // ending states an empty billed set, so settlement writes the performance sample every run // owes and no usage row. Measuring is not generating, and an upstream that answered the @@ -30,7 +30,7 @@ import { analyzeMessagesAffinity } from './affinity/ingress.ts'; import { renderMessagesError } from './errors.ts'; import { prepareMessagesWebSearchShimRequest } from './interceptors/web-search-shim.ts'; -import type { MessagesFacts } from './pipeline.ts'; +import type { AnthropicMessagesFacts } from './pipeline.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; @@ -47,22 +47,22 @@ import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; -import { parseAnthropicBetaHeader, type MessagesPayload } from '@floway-dev/protocols/messages'; +import { parseAnthropicBetaHeader, type AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import { providerModelOf } from '@floway-dev/provider'; /** Counting has no translation path: only an upstream's own Messages endpoint can answer the * question, so a candidate that would serve generation over a translated wire cannot serve * this. */ -export const messagesCountTokensTarget = chatTargetPicker(['messages']); +export const anthropicMessagesCountTokensTarget = chatTargetPicker(['messages']); /** What a measurement hands up. It rides at a protocol's own response key — which is what * lets the request rules generation runs be the same stages here — and the arm is narrowed * because nothing in a counting chain opens a stream. */ export type TokenCountAnswer = { readonly kind: 'value'; readonly body: unknown } | Failure; -type Counted = { [P in K]: TokenCountAnswer }; +type Counted = { [P in K]: TokenCountAnswer }; -type M = { [P in K]: MessagesFacts[P] }; +type M = { [P in K]: AnthropicMessagesFacts[P] }; /** * The outermost edge. A measurement is one object, so there is nothing to fold and nothing @@ -72,22 +72,22 @@ type M = { [P in K]: MessagesFacts[P] }; const emitMessagesTokenCount = defineStage< Record, Record, - Counted<'response.chat.messages'> & M<'response.http.headers'>, - M<'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers'>, + Counted<'response.chat.anthropicMessages'> & M<'response.http.headers'>, + M<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ name: 'emitMessagesTokenCount', through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.chat.messages', 'response.http.headers'], - consumes: ['response.chat.messages', 'response.http.headers'], - provides: ['response.chat.messages.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.chat.anthropicMessages', 'response.http.headers'], + consumes: ['response.chat.anthropicMessages', 'response.http.headers'], + provides: ['response.chat.anthropicMessages.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.chat.messages': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.chat.anthropicMessages': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -98,14 +98,14 @@ const emitMessagesTokenCount = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move(renderFailure(answer, () => renderMessagesError(answer.status, answer.message))), + 'response.chat.anthropicMessages.rendered': move(renderFailure(answer, () => renderMessagesError(answer.status, answer.message))), 'response.http.status': answer.status, }; } return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move(answer.body as Record), + 'response.chat.anthropicMessages.rendered': move(answer.body as Record), 'response.http.status': 200, }; }, @@ -128,40 +128,40 @@ const emitMessagesTokenCount = defineStage< * carries the `return` trait. */ const prepareMessagesWebSearchRequest = defineStage< - M<'request.chat.messages' | 'route.attempt'>, - M<'request.chat.messages'>, - Counted<'response.chat.messages'>, - Counted<'response.chat.messages'>, - Counted<'response.chat.messages'> & M<'response.usage.billable' | 'response.http.headers'>, + M<'request.chat.anthropicMessages' | 'route.attempt'>, + M<'request.chat.anthropicMessages'>, + Counted<'response.chat.anthropicMessages'>, + Counted<'response.chat.anthropicMessages'>, + Counted<'response.chat.anthropicMessages'> & M<'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'prepareMessagesWebSearchRequest', through: { request: { - needs: ['request.chat.messages', 'route.attempt'], + needs: ['request.chat.anthropicMessages', 'route.attempt'], consumes: [], - provides: ['request.chat.messages'], + provides: ['request.chat.anthropicMessages'], }, - response: { needs: ['response.chat.messages'], consumes: [], provides: [] }, + response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, }, - return: { provides: ['response.chat.messages', 'response.usage.billable', 'response.http.headers'] }, + return: { provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers'] }, execute: async (facts, next) => { if (!facts['route.attempt'].flags.includes('messages-web-search-shim')) return await next(facts); - const prepared = prepareMessagesWebSearchShimRequest(facts['request.chat.messages']); + const prepared = prepareMessagesWebSearchShimRequest(facts['request.chat.anthropicMessages']); if (prepared.type === 'invalid-request') { // Nothing was dialled, which is what an empty billed set and an empty header list say // on the other two keys. return move({ ...facts, - 'response.chat.messages': { status: 400, message: prepared.message }, + 'response.chat.anthropicMessages': { status: 400, message: prepared.message }, 'response.usage.billable': [], 'response.http.headers': [], }); } // A request that engaged nothing comes back by identity, so the record shows no change // where none happened. - if (prepared.payload === facts['request.chat.messages']) return await next(facts); - return await next({ ...facts, 'request.chat.messages': move(prepared.payload) }); + if (prepared.payload === facts['request.chat.anthropicMessages']) return await next(facts); + return await next({ ...facts, 'request.chat.anthropicMessages': move(prepared.payload) }); }, }); @@ -171,13 +171,13 @@ const prepareMessagesWebSearchRequest = defineStage< * never a stream. */ const callMessagesCountTokensUpstream = defineStage< - M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, - Counted<'response.chat.messages'> & M<'response.usage.billable' | 'response.http.headers'>, + M<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, + Counted<'response.chat.anthropicMessages'> & M<'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'callMessagesCountTokensUpstream', return: { - provides: ['response.chat.messages', 'response.usage.billable', 'response.http.headers'], + provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers'], }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -185,7 +185,7 @@ const callMessagesCountTokensUpstream = defineStage< // The payload affinity materialized for this candidate, as every stage between the fork // and here has rewritten it — built for the dial exactly as generation builds it, so what // is measured is what generation would be charged for. - const body = bodyForAttempt(facts['request.chat.messages'], candidate, applyRulesToUpstreamMessages); + const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamMessages); // Anthropic's beta flags have a typed path of their own, so no header allowlist can admit // them, and they are the client's own only when the client spoke this protocol. @@ -213,7 +213,7 @@ const callMessagesCountTokensUpstream = defineStage< return move({ ...facts, ...nothingBilled, - 'response.chat.messages': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.anthropicMessages': { status: 502, message: error instanceof Error ? error.message : String(error) }, 'response.http.headers': [], }); } @@ -227,7 +227,7 @@ const callMessagesCountTokensUpstream = defineStage< return move({ ...facts, ...nothingBilled, - 'response.chat.messages': { + 'response.chat.anthropicMessages': { status: response.status, message: text, ...(parsed === undefined ? {} : { body: parsed }), @@ -239,7 +239,7 @@ const callMessagesCountTokensUpstream = defineStage< return move({ ...facts, ...nothingBilled, - 'response.chat.messages': { kind: 'value' as const, body: parsed }, + 'response.chat.anthropicMessages': { kind: 'value' as const, body: parsed }, 'response.http.headers': [...response.headers], }); }, @@ -256,7 +256,7 @@ const callMessagesCountTokensUpstream = defineStage< * would have sent it, and the web-search preparation states the tool shape it would have * seen. */ -export const messagesCountTokensWire: readonly Stage[] = [ +export const anthropicMessagesCountTokensWire: readonly Stage[] = [ prepareMessagesWebSearchRequest, stripBillingAttributionFromMessages, disableReasoningOnForcedToolChoiceForMessages, @@ -267,35 +267,35 @@ export const messagesCountTokensWire: readonly Stage[] = [ /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: MessagesPayload): ChatNarrowing> => ({ - canServe: candidate => messagesCountTokensTarget.canServe(candidate.model.endpoints), +const narrowing = (payload: AnthropicMessagesPayload): ChatNarrowing> => ({ + canServe: candidate => anthropicMessagesCountTokensTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeMessagesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /messages/count_tokens endpoint.`, - refuse: (status, message) => ({ 'response.chat.messages': { status, message } }), - refuses: ['response.chat.messages'], + refuse: (status, message) => ({ 'response.chat.anthropicMessages': { status, message } }), + refuses: ['response.chat.anthropicMessages'], }); -export type MessagesCountTokensEntry = M< - 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.messages' | 'serve.model' +export type AnthropicMessagesCountTokensEntry = M< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.anthropicMessages' | 'serve.model' >; -export type MessagesCountTokensExit = M< - 'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers' +export type AnthropicMessagesCountTokensExit = M< + 'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers' >; -export const messagesCountTokensPipeline = (payload: MessagesPayload): Pipeline => - compose('messagesCountTokens', [ +export const anthropicMessagesCountTokensPipeline = (payload: AnthropicMessagesPayload): Pipeline => + compose('anthropicMessagesCountTokens', [ emitMessagesTokenCount, // A measurement goes through settlement like every other run. It provides an empty // billed set because nothing here is billable today, not because the operation is // exempt — an upstream that began charging for it would provide a non-empty one and // nothing else would change. - writeSettlement(handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages'])), + writeSettlement(handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages'])), resolveChatCandidates(narrowing(payload)), failover({ - failed: handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages']), + failed: handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), owns: [], }), - materializeAttempt('request.chat.messages'), - ...messagesCountTokensWire, + materializeAttempt('request.chat.anthropicMessages'), + ...anthropicMessagesCountTokensWire, ]); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts index 889d6b11b6..993434ddbe 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts @@ -1,4 +1,4 @@ -import { generateAnthropicId } from '@floway-dev/protocols/messages'; +import { generateAnthropicId } from '@floway-dev/protocols/anthropic-messages'; // Anthropic's own error envelope. The byte shape matches Anthropic-direct: // `{type:'error', error:{type, message}, request_id}` with `request_id` at the diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts index c73278e1c1..4eef5bfb47 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts @@ -6,20 +6,20 @@ // hand it over, and turn what the run answered with into a response. Everything between is // stages. -import { messagesCountTokensPipeline } from './count-tokens.ts'; +import { anthropicMessagesCountTokensPipeline } from './count-tokens.ts'; import { renderMessagesError } from './errors.ts'; -import { messagesKeepAlive, messagesServePipeline } from './pipeline.ts'; +import { anthropicMessagesKeepAlive, anthropicMessagesServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; -import { createNonResponsesSourceStore } from '../responses/items/store.ts'; +import { createNonResponsesSourceStore } from '../openai-responses/items/store.ts'; import { move } from '@floway-dev/pipeline'; -import type { MessagesPayload } from '@floway-dev/protocols/messages'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; // Reject `anthropic_beta` / `betas` in the body; the Messages protocol carries // them via the `anthropic-beta` HTTP header. -const rejectBodyBetaResponse = (payload: MessagesPayload): Response | null => { +const rejectBodyBetaResponse = (payload: AnthropicMessagesPayload): Response | null => { const record = payload as unknown as Record; const param = Object.hasOwn(record, 'anthropic_beta') ? 'anthropic_beta' @@ -41,9 +41,9 @@ const rejectBodyBetaResponse = (payload: MessagesPayload): Response | null => { /** A body the gateway could not read is reported in the words this protocol's own clients * parse, rather than as a fault of the gateway's. */ -const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: MessagesPayload } | { type: 'invalid'; message: string } => { +const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: AnthropicMessagesPayload } | { type: 'invalid'; message: string } => { try { - return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as MessagesPayload }; + return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as AnthropicMessagesPayload }; } catch (error) { return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } @@ -57,7 +57,7 @@ const refuse = (c: AuthedContext, ingress: Ingress, response: Response): Respons return finalizeGatewayResponse(refused.gateway, response); }; -export const messagesHttp = { +export const anthropicMessagesHttp = { generate: async (c: AuthedContext): Promise => { const ingress = await readIngress(c); const request = readRequest(ingress.body.bytes); @@ -79,22 +79,22 @@ export const messagesHttp = { return await serveThrough( c, prologue, - messagesServePipeline(payload), + anthropicMessagesServePipeline(payload), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'messages', - 'ingress.chat.messages.wantsStream': wantsStream, - 'request.chat.messages': payload, + 'ingress.chat.anthropicMessages.wantsStream': wantsStream, + 'request.chat.anthropicMessages': payload, 'serve.model': payload.model, }) as never, facts => { - const rendered = facts['response.chat.messages.rendered']; + const rendered = facts['response.chat.anthropicMessages.rendered']; // Anthropic defines a `ping` event and its clients read one, so an idle connection is // held open with that rather than with a comment no client sees. - if (isFrames(rendered)) return { frames: rendered, keepAlive: messagesKeepAlive }; + if (isFrames(rendered)) return { frames: rendered, keepAlive: anthropicMessagesKeepAlive }; return { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - facts => facts['response.chat.messages.streamedUsage'], + facts => facts['response.chat.anthropicMessages.streamedUsage'], ); }, @@ -118,15 +118,15 @@ export const messagesHttp = { return await serveThrough( c, prologue, - messagesCountTokensPipeline(payload), + anthropicMessagesCountTokensPipeline(payload), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'messages', - 'request.chat.messages': payload, + 'request.chat.anthropicMessages': payload, 'serve.model': payload.model, }) as never, // A measurement is one body however the turn went, so there is never a stream to write. - facts => ({ body: JSON.stringify(facts['response.chat.messages.rendered']), contentType: 'application/json' }), + facts => ({ body: JSON.stringify(facts['response.chat.anthropicMessages.rendered']), contentType: 'application/json' }), ); }, }; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts index b6a555d27d..7a8a61fff7 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts @@ -1,7 +1,7 @@ -import type { MessagesInterceptor } from './types.ts'; +import type { AnthropicMessagesInterceptor } from './types.ts'; import { telemetryModelIdentity } from '../../../shared/telemetry/attribution.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { generateAnthropicId, type MessagesPayload, type MessagesStreamEvent } from '@floway-dev/protocols/messages'; +import { generateAnthropicId, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { eventResult, providerModelOf, type ExecuteResult, type ModelCandidate } from '@floway-dev/provider'; // Claude Code answers "is this model usable?" by generating one token against @@ -68,7 +68,7 @@ const PROBE_PROMPTS: ReadonlySet = new Set(['Hi', 'test']); // The whole conversation of a probe. `model_validation` sends its prompt as a // single ephemeral text block and the credential check sends the bare-string // form; both shapes are current. Anything longer is a real turn. -const soleUserPromptOf = (payload: MessagesPayload): string | null => { +const soleUserPromptOf = (payload: AnthropicMessagesPayload): string | null => { if (payload.messages.length !== 1) return null; const message = payload.messages[0]!; if (message.role !== 'user') return null; @@ -79,7 +79,7 @@ const soleUserPromptOf = (payload: MessagesPayload): string | null => { return block.type === 'text' ? block.text : null; }; -export const isClaudeCodeProbe = (payload: MessagesPayload, headers: Headers): boolean => { +export const isClaudeCodeProbe = (payload: AnthropicMessagesPayload, headers: Headers): boolean => { if (payload.max_tokens !== 1) return false; // Every real Claude Code turn ships the session's tools; the one-token // probes never do. @@ -95,7 +95,7 @@ export const isClaudeCodeProbe = (payload: MessagesPayload, headers: Headers): b // usage block is load-bearing — the validation path dereferences // `usage.input_tokens` / `usage.output_tokens` unconditionally, and a missing // `usage` throws inside the CLI and reads as a failed probe. -export const probeFrames = async function* (model: string): AsyncGenerator> { +export const probeFrames = async function* (model: string): AsyncGenerator> { yield eventFrame({ type: 'message_start', message: { @@ -118,10 +118,10 @@ export const probeFrames = async function* (model: string): AsyncGenerator> => +const probeResult = (candidate: ModelCandidate, model: string): ExecuteResult> => eventResult(probeFrames(model), telemetryModelIdentity(candidate, providerModelOf(candidate).id)); -export const answerClaudeCodeProbe: MessagesInterceptor = async (ctx, _gatewayCtx, run) => +export const answerClaudeCodeProbe: AnthropicMessagesInterceptor = async (ctx, _gatewayCtx, run) => isClaudeCodeProbe(ctx.payload, ctx.headers) ? probeResult(ctx.candidate, ctx.payload.model) : await run(); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts index 82909f641f..44753e3fc5 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts @@ -1,23 +1,23 @@ import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; import type { Interceptor } from '@floway-dev/interceptor'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { MessagesStreamEvent } from '@floway-dev/protocols/messages'; -import type { ExecuteResult, MessagesInvocation } from '@floway-dev/provider'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ExecuteResult, AnthropicMessagesInvocation } from '@floway-dev/provider'; -export type { MessagesInvocation }; +export type { AnthropicMessagesInvocation }; -export type MessagesInterceptor = Interceptor< - MessagesInvocation, +export type AnthropicMessagesInterceptor = Interceptor< + AnthropicMessagesInvocation, GatewayCtx, - ExecuteResult> + ExecuteResult> >; // count_tokens is a one-shot, non-streaming HTTP exchange whose terminal // returns the raw upstream `Response`. Shared entries must therefore be pure // header/payload mutators; post-run stream inspection is not portable to this // result type. -export type MessagesCountTokensInterceptor = Interceptor< - MessagesInvocation, +export type AnthropicMessagesCountTokensInterceptor = Interceptor< + AnthropicMessagesInvocation, GatewayCtx, Response >; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 24420da575..5c3b7b6d30 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -8,7 +8,7 @@ // answerClaudeCodeProbe answers the CLI's model probe without dialling anything // dialChatWire the ending: picks this candidate's wire and hands into it // -// Three wires, all handing up `response.chat.messages`: this protocol's own, and the two +// Three wires, all handing up `response.chat.anthropicMessages`: this protocol's own, and the two // translated ones — Messages via Responses and Messages via Chat Completions — each a // handoff followed by that protocol's own wire. The stage above cannot tell which ran. // @@ -36,7 +36,7 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; -import { chatCompletionsWire } from '../chat-completions/pipeline.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { @@ -44,7 +44,7 @@ import { disableReasoningOnForcedToolChoiceForMessages, stripBillingAttributionFromMessages, } from '../interceptors.ts'; -import { responsesWire } from '../responses/pipeline.ts'; +import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; @@ -54,30 +54,30 @@ import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-d import { sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { collectMessagesProtocolEventsToResult, - messagesProtocolFrameToSSEFrame, + anthropicMessagesProtocolFrameToSSEFrame, MESSAGES_MISSING_TERMINAL_MESSAGE, parseAnthropicBetaHeader, - type MessagesPayload, - type MessagesStreamEvent, -} from '@floway-dev/protocols/messages'; + type AnthropicMessagesPayload, + type AnthropicMessagesStreamEvent, +} from '@floway-dev/protocols/anthropic-messages'; import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateMessagesViaChatCompletions, translateMessagesViaResponses } from '@floway-dev/translate'; /** `/v1/messages` prefers its own wire, then the translated Responses path, then the * translated Chat Completions path. */ -export const messagesTarget = chatTargetPicker(['messages', 'responses', 'chat-completions']); +export const anthropicMessagesTarget = chatTargetPicker(['messages', 'responses', 'chat-completions']); /** What this family adds to the chat space. */ -export interface MessagesFacts extends ChatFacts { +export interface AnthropicMessagesFacts extends ChatFacts { /** What the client is actually sent — an object when it asked for one, SSE frames when it * asked to stream. The edge provides it, so a dump shows what the client received. */ - 'response.chat.messages.rendered': Record | AsyncIterable; + 'response.chat.anthropicMessages.rendered': Record | AsyncIterable; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ - 'response.chat.messages.streamedUsage': Promise | null; + 'response.chat.anthropicMessages.streamedUsage': Promise | null; } -type M = { [P in K]: MessagesFacts[P] }; +type M = { [P in K]: AnthropicMessagesFacts[P] }; /** * The outermost edge. A Messages answer is always a stream by the time it reaches here — @@ -90,24 +90,24 @@ type M = { [P in K]: MessagesFacts[P] }; * ran out before `message_stop`, whichever of the two the client asked for. */ const emitMessages = defineStage< - M<'ingress.chat.messages.wantsStream'>, - M<'ingress.chat.messages.wantsStream'>, - M<'ingress.chat.messages.wantsStream' | 'response.chat.messages' | 'response.http.headers'>, - M<'response.chat.messages.rendered' | 'response.http.status' | 'response.http.headers'>, + M<'ingress.chat.anthropicMessages.wantsStream'>, + M<'ingress.chat.anthropicMessages.wantsStream'>, + M<'ingress.chat.anthropicMessages.wantsStream' | 'response.chat.anthropicMessages' | 'response.http.headers'>, + M<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ name: 'emitMessages', through: { - request: { needs: ['ingress.chat.messages.wantsStream'], consumes: [], provides: [] }, + request: { needs: ['ingress.chat.anthropicMessages.wantsStream'], consumes: [], provides: [] }, response: { - needs: ['response.chat.messages', 'response.http.headers'], - consumes: ['response.chat.messages', 'response.http.headers'], - provides: ['response.chat.messages.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.chat.anthropicMessages', 'response.http.headers'], + consumes: ['response.chat.anthropicMessages', 'response.http.headers'], + provides: ['response.chat.anthropicMessages.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next, use) => { const back = await next(facts); - const { 'response.chat.messages': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.chat.anthropicMessages': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -118,7 +118,7 @@ const emitMessages = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move(renderFailure(answer, () => renderMessagesError(answer.status, answer.message))), + 'response.chat.anthropicMessages.rendered': move(renderFailure(answer, () => renderMessagesError(answer.status, answer.message))), 'response.http.status': answer.status, }; } @@ -126,7 +126,7 @@ const emitMessages = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move(answer.body as Record), + 'response.chat.anthropicMessages.rendered': move(answer.body as Record), 'response.http.status': 200, }; } @@ -137,16 +137,16 @@ const emitMessages = defineStage< // the frames — below the fold, and there would be nothing left to rewrite. const frames = recordFrames( wrapMessagesAffinityEgress( - answer.frames as AsyncIterable>, + answer.frames as AsyncIterable>, affinityEgressOptions(use.gateway), ), use.gateway.dump, ); - if (!back['ingress.chat.messages.wantsStream']) { + if (!back['ingress.chat.anthropicMessages.wantsStream']) { return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move( + 'response.chat.anthropicMessages.rendered': move( await collectMessagesProtocolEventsToResult(frames) as unknown as Record, ), 'response.http.status': 200, @@ -155,7 +155,7 @@ const emitMessages = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.messages.rendered': move(renderSSE(frames)), + 'response.chat.anthropicMessages.rendered': move(renderSSE(frames)), 'response.http.status': 200, }; }, @@ -165,10 +165,10 @@ const emitMessages = defineStage< * frames have an SSE form at all is the protocol's to say, and it says so by writing no * frame — a stream terminator is a Chat Completions idea and there is nothing to send for * it here. */ -const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ [Symbol.asyncIterator]: () => (async function* () { for await (const frame of frames) { - const written = messagesProtocolFrameToSSEFrame(frame); + const written = anthropicMessagesProtocolFrameToSSEFrame(frame); if (written !== null) yield written; } })(), @@ -177,7 +177,7 @@ const renderSSE = (frames: AsyncIterable>): A /** * The native wire. It dials Messages and provides the answer at this family's own response * key — which is what will make it interchangeable with a translated chain: both hand up - * `response.chat.messages`, and the stage above cannot tell which ran. + * `response.chat.anthropicMessages`, and the stage above cannot tell which ran. */ /** * Answers Claude Code's one-token probe itself, rather than spending an upstream turn on it. @@ -194,40 +194,40 @@ const renderSSE = (frames: AsyncIterable>): A * family's own response keys. */ const answerClaudeCodeProbe = defineStage< - M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers'>, - M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers'>, - M<'response.chat.messages' | 'response.chat.messages.streamedUsage' + M<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers'>, + M<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers'>, + M<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, - M<'response.chat.messages' | 'response.chat.messages.streamedUsage' + M<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, - M<'response.chat.messages' | 'response.chat.messages.streamedUsage' + M<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'answerClaudeCodeProbe', through: { request: { - needs: ['request.chat.messages', 'route.attempt', 'ingress.http.headers'], + needs: ['request.chat.anthropicMessages', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: [], }, response: { - needs: ['response.chat.messages', 'response.http.headers'], + needs: ['response.chat.anthropicMessages', 'response.http.headers'], consumes: [], provides: [], }, }, return: { provides: [ - 'response.chat.messages', - 'response.chat.messages.streamedUsage', + 'response.chat.anthropicMessages', + 'response.chat.anthropicMessages.streamedUsage', 'response.usage.billable', 'response.http.headers', ], }, execute: async (facts, next, use) => { const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); - if (!isClaudeCodeProbe(facts['request.chat.messages'], headers)) return await next(facts); + if (!isClaudeCodeProbe(facts['request.chat.anthropicMessages'], headers)) return await next(facts); const candidate = use.resolveAttempt(facts['route.attempt']); // The probe is answered *for* this candidate, so it is the one a follow-up turn carrying @@ -236,13 +236,13 @@ const answerClaudeCodeProbe = defineStage< use.log.debug('answering a Claude Code probe without dialling', { upstream: facts['route.attempt'].upstreamId }); return move({ ...facts, - 'response.chat.messages': { + 'response.chat.anthropicMessages': { kind: 'stream' as const, - frames: probeFrames(facts['request.chat.messages'].model), + frames: probeFrames(facts['request.chat.anthropicMessages'].model), }, // Nothing streams from an upstream here, so there is nothing still to read: the row is // written now, at zero, which keeps the request visible without inventing latency. - 'response.chat.messages.streamedUsage': null, + 'response.chat.anthropicMessages.streamedUsage': null, 'response.usage.billable': [{ identity: telemetryModelIdentity(candidate, providerModelOf(candidate).id), quantities: {}, @@ -253,14 +253,14 @@ const answerClaudeCodeProbe = defineStage< }); const callMessagesUpstream = (streamedUsage: string) => defineStage< - M<'request.chat.messages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, - M<'response.chat.messages' | 'response.usage.billable' | 'response.http.headers'> & Record, + M<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, + M<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ name: 'callMessagesUpstream', return: { provides: [ - 'response.chat.messages', + 'response.chat.anthropicMessages', streamedUsage, 'response.usage.billable', 'response.http.headers', @@ -276,7 +276,7 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< // every stage between the fork and here has rewritten it — or, on a translated wire, what // the handoff put here. A thinking signature was rewritten for the upstream that will see // it, and one that no upstream but the issuer can read was dropped rather than sent on. - const body = bodyForAttempt(facts['request.chat.messages'], candidate, applyRulesToUpstreamMessages); + const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamMessages); // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump @@ -305,7 +305,7 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.chat.messages': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.anthropicMessages': { status: 502, message: error instanceof Error ? error.message : String(error) }, [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], @@ -324,7 +324,7 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } return move({ ...facts, - 'response.chat.messages': { + 'response.chat.anthropicMessages': { status: result.response.status, message: text, ...(parsed === undefined ? {} : { body: parsed }), @@ -341,7 +341,7 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< const metered = meterMessages(result.events, identity, use.gateway.attempt); return move({ ...facts, - 'response.chat.messages': { kind: 'stream' as const, frames: metered.frames }, + 'response.chat.anthropicMessages': { kind: 'stream' as const, frames: metered.frames }, [streamedUsage]: metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], @@ -358,7 +358,7 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< * endpoint accepts, so it applies to whatever body this wire actually sends and to nothing * that leaves for another protocol. */ -export const messagesWire = (streamedUsage: string): readonly Stage[] => [ +export const anthropicMessagesWire = (streamedUsage: string): readonly Stage[] => [ stripBillingAttributionFromMessages, disableReasoningOnForcedToolChoiceForMessages, applyRoleCompatibilityToMessages, @@ -366,31 +366,31 @@ export const messagesWire = (streamedUsage: string): readonly Stage[] => [ ]; /** This family's own reading, which every wire under it hands up. */ -const STREAMED_USAGE = 'response.chat.messages.streamedUsage'; +const STREAMED_USAGE = 'response.chat.anthropicMessages.streamedUsage'; /** The three wires `/v1/messages` can be served on. Its own is the bare wire; each translated * one is a handoff and then the target protocol's own wire. */ -const messagesWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { +const anthropicMessagesWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { switch (target) { case 'messages': - return compose('messagesNative', messagesWire(STREAMED_USAGE)); + return compose('anthropicMessagesNative', anthropicMessagesWire(STREAMED_USAGE)); case 'responses': - return compose('messagesViaResponses', [ + return compose('anthropicMessagesViaOpenAIResponses', [ handOff({ - from: { request: 'request.chat.messages', response: 'response.chat.messages' }, - to: { request: 'request.chat.responses', response: 'response.chat.responses' }, + from: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, trip: async payload => await translateMessagesViaResponses(payload, { model: candidate.model.id }), }), - ...responsesWire(STREAMED_USAGE), + ...openaiResponsesWire(STREAMED_USAGE), ]); case 'chat-completions': - return compose('messagesViaChatCompletions', [ + return compose('anthropicMessagesViaOpenAIChatCompletions', [ handOff({ - from: { request: 'request.chat.messages', response: 'response.chat.messages' }, - to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + from: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, trip: async payload => await translateMessagesViaChatCompletions(payload, { model: candidate.model.id }), }), - ...chatCompletionsWire(STREAMED_USAGE), + ...openaiChatCompletionsWire(STREAMED_USAGE), ]); } }; @@ -400,10 +400,10 @@ const messagesWireFor = (target: ChatTargetApi, candidate: ModelCandidate): Chat * `message_start` and output accounting on `message_delta`, so the reader that merges the * two is per-stream state and is made here rather than shared. */ const meterMessages = ( - source: AsyncIterable>, + source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, -): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { +): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { let settle!: (outcome: StreamOutcome) => void; const outcome = new Promise(resolve => { settle = resolve; }); // Running out without the terminal frame is what "it did not finish" means, and it is known @@ -445,7 +445,7 @@ const meterMessages = ( /** What ends a Messages turn. Anthropic's own stream terminator is an event rather than a * transport sentinel, and a stream that failed mid-turn says so with `error` in place of the * `message_stop` that will now never come. */ -const isMessagesTerminalFrame = (frame: ProtocolFrame): boolean => +const isMessagesTerminalFrame = (frame: ProtocolFrame): boolean => frame.type === 'event' && (frame.event.type === 'message_stop' || frame.event.type === 'error'); /** What one attempt is billable for. An upstream that reported nothing leaves no quantities @@ -462,53 +462,53 @@ const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModel /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: MessagesPayload): ChatNarrowing> => ({ - canServe: candidate => messagesTarget.canServe(candidate.model.endpoints), +const narrowing = (payload: AnthropicMessagesPayload): ChatNarrowing> => ({ + canServe: candidate => anthropicMessagesTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeMessagesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /messages endpoint.`, refuse: (status, message) => ({ - 'response.chat.messages': { status, message }, + 'response.chat.anthropicMessages': { status, message }, // A refusal never opened a stream, so there is nothing still to read — which is what // lets settlement write its row here rather than wait for numbers that never come. - 'response.chat.messages.streamedUsage': null, + 'response.chat.anthropicMessages.streamedUsage': null, }), - refuses: ['response.chat.messages', 'response.chat.messages.streamedUsage'], + refuses: ['response.chat.anthropicMessages', 'response.chat.anthropicMessages.streamedUsage'], }); /** What this protocol writes on an idle connection. Anthropic defines a `ping` event and * clients read it, so an SSE comment — invisible by design — is not the same wire byte. * The route that serves this chain hands it to the seam alongside the frames. */ -export const messagesKeepAlive: SseWritableFrame = sseFrame(JSON.stringify({ type: 'ping' }), 'ping'); +export const anthropicMessagesKeepAlive: SseWritableFrame = sseFrame(JSON.stringify({ type: 'ping' }), 'ping'); -export type MessagesServeEntry = M< - 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.messages.wantsStream' - | 'request.chat.messages' | 'serve.model' +export type AnthropicMessagesServeEntry = M< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.anthropicMessages.wantsStream' + | 'request.chat.anthropicMessages' | 'serve.model' >; -export type MessagesServeExit = M< - 'response.chat.messages.rendered' | 'response.chat.messages.streamedUsage' +export type AnthropicMessagesServeExit = M< + 'response.chat.anthropicMessages.rendered' | 'response.chat.anthropicMessages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' >; -export const messagesServePipeline = (payload: MessagesPayload): Pipeline => - compose('messagesServe', [ +export const anthropicMessagesServePipeline = (payload: AnthropicMessagesPayload): Pipeline => + compose('anthropicMessagesServe', [ emitMessages, writeSettlement( - handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages']), - handedUp => (handedUp as { 'response.chat.messages.streamedUsage'?: unknown })['response.chat.messages.streamedUsage'] !== null, + handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), + handedUp => (handedUp as { 'response.chat.anthropicMessages.streamedUsage'?: unknown })['response.chat.anthropicMessages.streamedUsage'] !== null, ), resolveChatCandidates(narrowing(payload)), failover({ - failed: handedUp => isFailure((handedUp as { 'response.chat.messages'?: unknown })['response.chat.messages']), + failed: handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), owns: [], }), - materializeAttempt('request.chat.messages'), + materializeAttempt('request.chat.anthropicMessages'), answerClaudeCodeProbe, dialChatWire({ - source: 'request.chat.messages', - needs: ['request.chat.messages', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.messages', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], - pick: endpoints => messagesTarget.pick(endpoints), - wire: messagesWireFor, + source: 'request.chat.anthropicMessages', + needs: ['request.chat.anthropicMessages', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.anthropicMessages', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => anthropicMessagesTarget.pick(endpoints), + wire: anthropicMessagesWireFor, }), ]); diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts index 5f76cc8ca6..8bb413ccb8 100644 --- a/packages/gateway/src/data-plane/chat/facts.ts +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -11,10 +11,10 @@ // `wantsStream` is four disjoint keys rather than one shared one. import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; -import type { GeminiPayload } from '@floway-dev/protocols/gemini'; -import type { MessagesPayload } from '@floway-dev/protocols/messages'; -import type { ResponsesPayload } from '@floway-dev/protocols/responses'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; +import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; /** The four protocols a client can speak here. Gemini is source-only: nothing translates * into it, so by the ruling it contributes one pipeline rather than two — a source role @@ -31,27 +31,27 @@ export interface ChatFacts extends GatewayFacts { 'ingress.chat.sourceProtocol': ChatSourceProtocol; /** What the client asked for, per protocol and disjoint, because a translated request * must not inherit the target protocol's answer to a question the client never asked. */ - 'ingress.chat.chatCompletions.wantsStream': boolean; - 'ingress.chat.messages.wantsStream': boolean; - 'ingress.chat.responses.wantsStream': boolean; - 'ingress.chat.gemini.wantsStream': boolean; + 'ingress.chat.openaiChatCompletions.wantsStream': boolean; + 'ingress.chat.anthropicMessages.wantsStream': boolean; + 'ingress.chat.openaiResponses.wantsStream': boolean; + 'ingress.chat.geminiGenerateContent.wantsStream': boolean; /** Whether the client wants the usage chunk it would otherwise never see. The gateway * always asks the upstream for one; this decides who is shown it. */ - 'ingress.chat.chatCompletions.wantsUsageChunk': boolean; + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': boolean; /** One key per protocol. A translation consumes one and provides another, which is the * whole of what a protocol handoff is. */ - 'request.chat.chatCompletions': ChatCompletionsPayload; - 'request.chat.messages': MessagesPayload; - 'request.chat.responses': ResponsesPayload; - 'request.chat.gemini': GeminiPayload; + 'request.chat.openaiChatCompletions': OpenAIChatCompletionsPayload; + 'request.chat.anthropicMessages': AnthropicMessagesPayload; + 'request.chat.openaiResponses': OpenAIResponsesPayload; + 'request.chat.geminiGenerateContent': GeminiGenerateContentPayload; /** A stream, a value and a failure sit at one key, so telling them apart is reading a * value and no declaration ever mentions which arm is there. */ - 'response.chat.chatCompletions': ChatAnswer; - 'response.chat.messages': ChatAnswer; - 'response.chat.responses': ChatAnswer; - 'response.chat.gemini': ChatAnswer; + 'response.chat.openaiChatCompletions': ChatAnswer; + 'response.chat.anthropicMessages': ChatAnswer; + 'response.chat.openaiResponses': ChatAnswer; + 'response.chat.geminiGenerateContent': ChatAnswer; } /** What comes back at a protocol's response key. The frame type is the protocol's own; the diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index 7ad0899ebd..5c0ddbfb5f 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -10,7 +10,7 @@ // the Messages count wire the ending, shared with `/v1/messages/count_tokens` // // A second **operation** over this protocol rather than another wire under -// `geminiServePipeline`, so it is a chain of its own. Like generation it has no wire of its +// `geminiGenerateContentServePipeline`, so it is a chain of its own. Like generation it has no wire of its // own — the provider surface has no Gemini call — but unlike generation it has only one // target: no protocol but Messages answers "what would this cost", so there is nothing to // pick between and the pair is in the array rather than under a fork. @@ -27,7 +27,7 @@ import { analyzeGeminiAffinity } from './affinity/ingress.ts'; import { renderGeminiError } from './errors.ts'; -import type { GeminiFacts } from './pipeline.ts'; +import type { GeminiGenerateContentFacts } from './pipeline.ts'; import { asJsonObject, readJsonNumber } from '../../../shared/json-helpers.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; @@ -38,20 +38,20 @@ import { stripUnsupportedPartFieldsFromGemini, stripUnsupportedToolsFromGemini, } from '../interceptors.ts'; -import { messagesCountTokensWire, type TokenCountAnswer } from '../messages/count-tokens.ts'; +import { anthropicMessagesCountTokensWire, type TokenCountAnswer } from '../anthropic-messages/count-tokens.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; -import type { GeminiPayload } from '@floway-dev/protocols/gemini'; +import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; import { translateGeminiViaMessages, TranslatorInputError } from '@floway-dev/translate'; /** Counting is reachable only over an upstream's own Messages endpoint: no other protocol * answers the question, and no translation invents an answer. */ -export const geminiCountTokensTarget = chatTargetPicker(['messages']); +export const geminiGenerateContentCountTokensTarget = chatTargetPicker(['messages']); -type Counted = { [P in K]: TokenCountAnswer }; +type Counted = { [P in K]: TokenCountAnswer }; -type G = { [P in K]: GeminiFacts[P] }; +type G = { [P in K]: GeminiGenerateContentFacts[P] }; /** * The outermost edge. A measurement is one object, so there is nothing to fold and nothing to @@ -61,22 +61,22 @@ type G = { [P in K]: GeminiFacts[P] }; const emitGeminiTokenCount = defineStage< Record, Record, - Counted<'response.chat.gemini'> & G<'response.http.headers'>, - G<'response.chat.gemini.rendered' | 'response.http.status' | 'response.http.headers'>, + Counted<'response.chat.geminiGenerateContent'> & G<'response.http.headers'>, + G<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ name: 'emitGeminiTokenCount', through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.chat.gemini', 'response.http.headers'], - consumes: ['response.chat.gemini', 'response.http.headers'], - provides: ['response.chat.gemini.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.chat.geminiGenerateContent', 'response.http.headers'], + consumes: ['response.chat.geminiGenerateContent', 'response.http.headers'], + provides: ['response.chat.geminiGenerateContent.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.chat.gemini': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.chat.geminiGenerateContent': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -87,14 +87,14 @@ const emitGeminiTokenCount = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(renderFailure(answer, () => renderGeminiError(answer.status, answer.message))), + 'response.chat.geminiGenerateContent.rendered': move(renderFailure(answer, () => renderGeminiError(answer.status, answer.message))), 'response.http.status': answer.status, }; } return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(answer.body as Record), + 'response.chat.geminiGenerateContent.rendered': move(answer.body as Record), 'response.http.status': 200, }; }, @@ -111,30 +111,30 @@ const emitGeminiTokenCount = defineStage< * protocol cannot represent is answered here rather than measured. */ const measureGeminiAsMessages = defineStage< - G<'request.chat.gemini' | 'route.attempt'>, - G<'request.chat.messages'>, - Counted<'response.chat.messages'>, - Counted<'response.chat.gemini'>, - Counted<'response.chat.gemini'> & G<'response.usage.billable' | 'response.http.headers'>, + G<'request.chat.geminiGenerateContent' | 'route.attempt'>, + G<'request.chat.anthropicMessages'>, + Counted<'response.chat.anthropicMessages'>, + Counted<'response.chat.geminiGenerateContent'>, + Counted<'response.chat.geminiGenerateContent'> & G<'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'measureGeminiAsMessages', through: { request: { - needs: ['request.chat.gemini', 'route.attempt'], - consumes: ['request.chat.gemini'], - provides: ['request.chat.messages'], + needs: ['request.chat.geminiGenerateContent', 'route.attempt'], + consumes: ['request.chat.geminiGenerateContent'], + provides: ['request.chat.anthropicMessages'], }, response: { - needs: ['response.chat.messages'], - consumes: ['response.chat.messages'], - provides: ['response.chat.gemini'], + needs: ['response.chat.anthropicMessages'], + consumes: ['response.chat.anthropicMessages'], + provides: ['response.chat.geminiGenerateContent'], }, }, - return: { provides: ['response.chat.gemini', 'response.usage.billable', 'response.http.headers'] }, + return: { provides: ['response.chat.geminiGenerateContent', 'response.usage.billable', 'response.http.headers'] }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - const { 'request.chat.gemini': asked, ...down } = facts; + const { 'request.chat.geminiGenerateContent': asked, ...down } = facts; let trip; try { @@ -150,7 +150,7 @@ const measureGeminiAsMessages = defineStage< if (!(error instanceof TranslatorInputError)) throw error; return move({ ...down, - 'response.chat.gemini': { status: 400, message: error.message }, + 'response.chat.geminiGenerateContent': { status: 400, message: error.message }, // Nothing was measured, which is what an empty billed set and an empty header list say // on the other two keys. There is no streamed reading to leave behind: a measurement // never opens a stream, which is why this chain carries no such key at all. @@ -163,9 +163,9 @@ const measureGeminiAsMessages = defineStage< // answer to deliver here, so the field does not travel. const { stream: _stream, ...target } = trip.target; - const back = await next({ ...down, 'request.chat.messages': move(target) }); - const { 'response.chat.messages': counted, ...up } = back; - return { ...up, 'response.chat.gemini': move(asGeminiTokenCount(counted)) }; + const back = await next({ ...down, 'request.chat.anthropicMessages': move(target) }); + const { 'response.chat.anthropicMessages': counted, ...up } = back; + return { ...up, 'response.chat.geminiGenerateContent': move(asGeminiTokenCount(counted)) }; }, }); @@ -195,39 +195,39 @@ const asGeminiTokenCount = (counted: TokenCountAnswer): TokenCountAnswer => { /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: GeminiPayload): ChatNarrowing> => ({ - canServe: candidate => geminiCountTokensTarget.canServe(candidate.model.endpoints), +const narrowing = (payload: GeminiGenerateContentPayload): ChatNarrowing> => ({ + canServe: candidate => geminiGenerateContentCountTokensTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeGeminiAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support countTokens.`, - refuse: (status, message) => ({ 'response.chat.gemini': { status, message } }), - refuses: ['response.chat.gemini'], + refuse: (status, message) => ({ 'response.chat.geminiGenerateContent': { status, message } }), + refuses: ['response.chat.geminiGenerateContent'], }); -export type GeminiCountTokensEntry = G< - 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.gemini' | 'serve.model' +export type GeminiGenerateContentCountTokensEntry = G< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.geminiGenerateContent' | 'serve.model' >; -export type GeminiCountTokensExit = G< - 'response.chat.gemini.rendered' | 'response.http.status' | 'response.http.headers' +export type GeminiGenerateContentCountTokensExit = G< + 'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers' >; -export const geminiCountTokensPipeline = (payload: GeminiPayload): Pipeline => - compose('geminiCountTokens', [ +export const geminiGenerateContentCountTokensPipeline = (payload: GeminiGenerateContentPayload): Pipeline => + compose('geminiGenerateContentCountTokens', [ emitGeminiTokenCount, // A measurement goes through settlement like every other run. It provides an empty // billed set because nothing here is billable today, not because the operation is // exempt — an upstream that began charging for it would provide a non-empty one and // nothing else would change. - writeSettlement(handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini'])), + writeSettlement(handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent'])), resolveChatCandidates(narrowing(payload)), failover({ - failed: handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini']), + failed: handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), owns: [], }), - materializeAttempt('request.chat.gemini'), + materializeAttempt('request.chat.geminiGenerateContent'), stripUnsupportedPartFieldsFromGemini, stripUnsupportedToolsFromGemini, stripSafetySettingsFromGemini, measureGeminiAsMessages, - ...messagesCountTokensWire, + ...anthropicMessagesCountTokensWire, ]); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts index c69c92c6a2..8a17592b2b 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts @@ -4,10 +4,10 @@ * Google-RPC name and clients read it, so this shape is the protocol's rather than the * gateway's — an OpenAI-shaped envelope would leave that field undefined. */ export const renderGeminiError = (status: number, message: string): Record => ({ - error: { code: status, message, status: geminiStatusForHttpStatus(status) }, + error: { code: status, message, status: geminiGenerateContentStatusForHttpStatus(status) }, }); -export const geminiStatusForHttpStatus = (status: number): string => { +export const geminiGenerateContentStatusForHttpStatus = (status: number): string => { switch (status) { case 400: return 'INVALID_ARGUMENT'; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts index 7a86de043d..115a5556d0 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts @@ -6,18 +6,18 @@ // Each entry is a prologue and an epilogue around that chain: read what the client sent, hand // it over, and turn what the run answered with into a response. Everything between is stages. -import { geminiCountTokensPipeline } from './count-tokens.ts'; +import { geminiGenerateContentCountTokensPipeline } from './count-tokens.ts'; import { renderGeminiError } from './errors.ts'; -import { geminiServePipeline } from './pipeline.ts'; +import { geminiGenerateContentServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; -import { createNonResponsesSourceStore } from '../responses/items/store.ts'; +import { createNonResponsesSourceStore } from '../openai-responses/items/store.ts'; import { move } from '@floway-dev/pipeline'; -import type { GeminiContent, GeminiPayload } from '@floway-dev/protocols/gemini'; +import type { GeminiGenerateContentContent, GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -interface GeminiModelAction { +interface GeminiGenerateContentModelAction { readonly model: string; readonly action: string; } @@ -32,7 +32,7 @@ interface GeminiModelAction { * attempt to record and nothing for a run to hold beyond the refusal. */ const unknownAction = (message: string): Response => Response.json(renderGeminiError(404, message), { status: 404 }); -const parseGeminiModelAction = (modelAction: string | undefined): GeminiModelAction | Response => { +const parseGeminiModelAction = (modelAction: string | undefined): GeminiGenerateContentModelAction | Response => { if (!modelAction) return unknownAction('Missing Gemini model action.'); const separator = modelAction.lastIndexOf(':'); if (separator <= 0 || separator === modelAction.length - 1) return unknownAction(`Unknown Gemini model action: ${modelAction}`); @@ -41,9 +41,9 @@ const parseGeminiModelAction = (modelAction: string | undefined): GeminiModelAct // `:countTokens` can carry either `contents` directly or a nested // `generateContentRequest` envelope (Google's SDK shape). Normalize both to a -// single `GeminiPayload` for the rest of the chain. -const parseGeminiCountTokensPayload = (body: unknown): GeminiPayload => { - const shape = (body ?? {}) as { contents?: GeminiContent[]; generateContentRequest?: GeminiPayload }; +// single `GeminiGenerateContentPayload` for the rest of the chain. +const parseGeminiCountTokensPayload = (body: unknown): GeminiGenerateContentPayload => { + const shape = (body ?? {}) as { contents?: GeminiGenerateContentContent[]; generateContentRequest?: GeminiGenerateContentPayload }; return shape.generateContentRequest ?? { contents: shape.contents }; }; @@ -51,7 +51,7 @@ const parseGeminiCountTokensPayload = (body: unknown): GeminiPayload => { // once, then dispatches to the matching sub-handler. Keeping the parse here // means the sub-handlers see a validated `(model, action)` pair and never // need to re-emit "Unknown Gemini model action" on already-validated input. -export const geminiHttp = async (c: AuthedContext): Promise => { +export const geminiGenerateContentHttp = async (c: AuthedContext): Promise => { const parsed = parseGeminiModelAction(c.req.param('modelAction')); if (parsed instanceof Response) return parsed; if (parsed.action === 'countTokens') return await runGeminiCountTokens(c, parsed.model); @@ -82,7 +82,7 @@ const refuse = (c: AuthedContext, ingress: Ingress, message: string): Response = const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: boolean): Promise => { const ingress = await readIngress(c); - const request = readRequest(ingress.body.bytes, (body): GeminiPayload => body as GeminiPayload); + const request = readRequest(ingress.body.bytes, (body): GeminiGenerateContentPayload => body as GeminiGenerateContentPayload); if (request.type === 'invalid') return refuse(c, ingress, request.message); const { payload } = request; @@ -95,22 +95,22 @@ const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: b return await serveThrough( c, prologue, - geminiServePipeline(payload), + geminiGenerateContentServePipeline(payload), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'gemini', - 'ingress.chat.gemini.wantsStream': wantsStream, - 'request.chat.gemini': payload, + 'ingress.chat.geminiGenerateContent.wantsStream': wantsStream, + 'request.chat.geminiGenerateContent': payload, // Gemini carries the model in the path rather than the body, so the id the run // resolves against is the one the route split off. 'serve.model': model, }) as never, facts => { - const rendered = facts['response.chat.gemini.rendered']; + const rendered = facts['response.chat.geminiGenerateContent.rendered']; if (isFrames(rendered)) return { frames: rendered }; return { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - facts => facts['response.chat.gemini.streamedUsage'], + facts => facts['response.chat.geminiGenerateContent.streamedUsage'], ); }; @@ -129,14 +129,14 @@ const runGeminiCountTokens = async (c: AuthedContext, model: string): Promise ({ body: JSON.stringify(facts['response.chat.gemini.rendered']), contentType: 'application/json' }), + facts => ({ body: JSON.stringify(facts['response.chat.geminiGenerateContent.rendered']), contentType: 'application/json' }), ); }; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index 549520d313..10dbaabbef 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -10,7 +10,7 @@ // // Gemini is source-only. Nothing translates *into* it, so it contributes one pipeline rather // than two — a source role and no target role, and no other family's chain ever hands up -// `response.chat.gemini`. +// `response.chat.geminiGenerateContent`. // // It is also the one family with no wire of its own: the provider surface has no `callGemini`, // so every candidate is reached through a translation and all three wires are translated ones @@ -56,7 +56,7 @@ import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; -import { chatCompletionsWire } from '../chat-completions/pipeline.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { @@ -65,8 +65,8 @@ import { stripUnsupportedToolsFromGemini, suppressThoughtPartsFromGemini, } from '../interceptors.ts'; -import { messagesWire } from '../messages/pipeline.ts'; -import { responsesWire } from '../responses/pipeline.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; +import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; @@ -74,30 +74,30 @@ import { compose, defineStage, move, transform, type Pipeline } from '@floway-de import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { collectGeminiProtocolEventsToResult, - geminiProtocolFrameToSSEFrame, + geminiGenerateContentProtocolFrameToSSEFrame, GEMINI_MISSING_TERMINAL_MESSAGE, isGeminiTerminalEvent, - type GeminiPayload, - type GeminiStreamEvent, -} from '@floway-dev/protocols/gemini'; + type GeminiGenerateContentPayload, + type GeminiGenerateContentStreamEvent, +} from '@floway-dev/protocols/gemini-generate-content'; import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; import { translateGeminiViaChatCompletions, translateGeminiViaMessages, translateGeminiViaResponses } from '@floway-dev/translate'; /** `:generateContent` has no wire of its own, so the whole preference list is translated: * Chat Completions first, then Messages, then Responses. */ -export const geminiTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); +export const geminiGenerateContentTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); /** What this family adds to the chat space. */ -export interface GeminiFacts extends ChatFacts { +export interface GeminiGenerateContentFacts extends ChatFacts { /** What the client is actually sent — an object when it asked for one, SSE frames when it * asked to stream. The edge provides it, so a dump shows what the client received. */ - 'response.chat.gemini.rendered': Record | AsyncIterable; + 'response.chat.geminiGenerateContent.rendered': Record | AsyncIterable; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ - 'response.chat.gemini.streamedUsage': Promise | null; + 'response.chat.geminiGenerateContent.streamedUsage': Promise | null; } -type C = { [P in K]: GeminiFacts[P] }; +type C = { [P in K]: GeminiGenerateContentFacts[P] }; /** * The outermost edge. A Gemini answer is always a stream by the time it reaches here — the @@ -109,28 +109,28 @@ type C = { [P in K]: GeminiFacts[P] }; * same frames that would have gone out are folded here instead. */ const emitGemini = defineStage< - C<'ingress.chat.gemini.wantsStream'>, - C<'ingress.chat.gemini.wantsStream'>, - C<'ingress.chat.gemini.wantsStream' | 'response.chat.gemini' | 'response.http.headers'>, - C<'response.chat.gemini.rendered' | 'response.http.status' | 'response.http.headers'>, + C<'ingress.chat.geminiGenerateContent.wantsStream'>, + C<'ingress.chat.geminiGenerateContent.wantsStream'>, + C<'ingress.chat.geminiGenerateContent.wantsStream' | 'response.chat.geminiGenerateContent' | 'response.http.headers'>, + C<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ name: 'emitGemini', through: { request: { - needs: ['ingress.chat.gemini.wantsStream'], + needs: ['ingress.chat.geminiGenerateContent.wantsStream'], consumes: [], provides: [], }, response: { - needs: ['response.chat.gemini', 'response.http.headers'], - consumes: ['response.chat.gemini', 'response.http.headers'], - provides: ['response.chat.gemini.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.chat.geminiGenerateContent', 'response.http.headers'], + consumes: ['response.chat.geminiGenerateContent', 'response.http.headers'], + provides: ['response.chat.geminiGenerateContent.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next, use) => { const back = await next(facts); - const { 'response.chat.gemini': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.chat.geminiGenerateContent': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -141,7 +141,7 @@ const emitGemini = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(renderFailure(answer, () => renderGeminiError(answer.status, answer.message))), + 'response.chat.geminiGenerateContent.rendered': move(renderFailure(answer, () => renderGeminiError(answer.status, answer.message))), 'response.http.status': answer.status, }; } @@ -149,7 +149,7 @@ const emitGemini = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(answer.body as Record), + 'response.chat.geminiGenerateContent.rendered': move(answer.body as Record), 'response.http.status': 200, }; } @@ -160,16 +160,16 @@ const emitGemini = defineStage< // the frames — below the fold, and there would be nothing left to rewrite. const frames = recordFrames( wrapGeminiAffinityEgress( - answer.frames as AsyncIterable>, + answer.frames as AsyncIterable>, affinityEgressOptions(use.gateway), ), use.gateway.dump, ); - if (!back['ingress.chat.gemini.wantsStream']) { + if (!back['ingress.chat.geminiGenerateContent.wantsStream']) { return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move( + 'response.chat.geminiGenerateContent.rendered': move( await collectGeminiProtocolEventsToResult(frames) as unknown as Record, ), 'response.http.status': 200, @@ -178,7 +178,7 @@ const emitGemini = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.gemini.rendered': move(renderSSE(frames)), + 'response.chat.geminiGenerateContent.rendered': move(renderSSE(frames)), 'response.http.status': 200, }; }, @@ -186,10 +186,10 @@ const emitGemini = defineStage< /** Gemini streams one JSON object per `data:` line and has no sentinel to write, which the * protocol says by writing no frame at all for the one that ends the stream. */ -const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ [Symbol.asyncIterator]: () => (async function* () { for await (const frame of frames) { - const written = geminiProtocolFrameToSSEFrame(frame); + const written = geminiGenerateContentProtocolFrameToSSEFrame(frame); if (written !== null) yield written; } })(), @@ -211,28 +211,28 @@ const renderSSE = (frames: AsyncIterable>): Asy const requireGeminiTerminal = defineStage< Record, Record, - C<'response.chat.gemini'>, - C<'response.chat.gemini'> + C<'response.chat.geminiGenerateContent'>, + C<'response.chat.geminiGenerateContent'> >({ name: 'requireGeminiTerminal', through: { request: { needs: [], consumes: [], provides: [] }, - response: { needs: ['response.chat.gemini'], consumes: [], provides: ['response.chat.gemini'] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: ['response.chat.geminiGenerateContent'] }, }, execute: transform< Record, Record, - C<'response.chat.gemini'>, - C<'response.chat.gemini'> + C<'response.chat.geminiGenerateContent'>, + C<'response.chat.geminiGenerateContent'> >(() => ({ response: facts => { - const answer = facts['response.chat.gemini']; + const answer = facts['response.chat.geminiGenerateContent']; // A refusal and a collected body never opened a stream, so there is no end to find. if (isFailure(answer) || answer.kind !== 'stream') return facts; - const frames = answer.frames as AsyncIterable>; + const frames = answer.frames as AsyncIterable>; return { ...facts, - 'response.chat.gemini': move({ + 'response.chat.geminiGenerateContent': move({ kind: 'stream' as const, frames: { [Symbol.asyncIterator]: () => framesUntilTerminal(frames) }, }), @@ -245,8 +245,8 @@ const requireGeminiTerminal = defineStage< * here closes the translation and the wire under it — and fails a stream that ran out before * one arrived. */ const framesUntilTerminal = async function* ( - frames: AsyncIterable>, -): AsyncGenerator> { + frames: AsyncIterable>, +): AsyncGenerator> { for await (const frame of frames) { yield frame; if (frame.type === 'done' || isGeminiTerminalEvent(frame.event)) return; @@ -255,39 +255,39 @@ const framesUntilTerminal = async function* ( }; /** This family's own reading, which every wire under it hands up. */ -const STREAMED_USAGE = 'response.chat.gemini.streamedUsage'; +const STREAMED_USAGE = 'response.chat.geminiGenerateContent.streamedUsage'; /** The three wires `:generateContent` can be served on, and all three are translated ones: * the provider surface has no Gemini call, so a handoff is how every candidate is reached. */ -const geminiWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { +const geminiGenerateContentWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { const context = { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens }; switch (target) { case 'chat-completions': - return compose('geminiViaChatCompletions', [ + return compose('geminiGenerateContentViaOpenAIChatCompletions', [ handOff({ - from: { request: 'request.chat.gemini', response: 'response.chat.gemini' }, - to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, + to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, trip: async payload => await translateGeminiViaChatCompletions(payload, context), }), - ...chatCompletionsWire(STREAMED_USAGE), + ...openaiChatCompletionsWire(STREAMED_USAGE), ]); case 'messages': - return compose('geminiViaMessages', [ + return compose('geminiGenerateContentViaAnthropicMessages', [ handOff({ - from: { request: 'request.chat.gemini', response: 'response.chat.gemini' }, - to: { request: 'request.chat.messages', response: 'response.chat.messages' }, + from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, + to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, trip: async payload => await translateGeminiViaMessages(payload, context), }), - ...messagesWire(STREAMED_USAGE), + ...anthropicMessagesWire(STREAMED_USAGE), ]); case 'responses': - return compose('geminiViaResponses', [ + return compose('geminiGenerateContentViaOpenAIResponses', [ handOff({ - from: { request: 'request.chat.gemini', response: 'response.chat.gemini' }, - to: { request: 'request.chat.responses', response: 'response.chat.responses' }, + from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, + to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, trip: async payload => await translateGeminiViaResponses(payload, context), }), - ...responsesWire(STREAMED_USAGE), + ...openaiResponsesWire(STREAMED_USAGE), ]); } }; @@ -295,52 +295,52 @@ const geminiWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWi /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: GeminiPayload): ChatNarrowing> => ({ - canServe: candidate => geminiTarget.canServe(candidate.model.endpoints), +const narrowing = (payload: GeminiGenerateContentPayload): ChatNarrowing> => ({ + canServe: candidate => geminiGenerateContentTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeGeminiAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the Gemini generateContent endpoint.`, refuse: (status, message) => ({ - 'response.chat.gemini': { status, message }, + 'response.chat.geminiGenerateContent': { status, message }, // A refusal never opened a stream, so there is nothing still to read — which is what // lets settlement write its row here rather than wait for numbers that never come. - 'response.chat.gemini.streamedUsage': null, + 'response.chat.geminiGenerateContent.streamedUsage': null, }), - refuses: ['response.chat.gemini', 'response.chat.gemini.streamedUsage'], + refuses: ['response.chat.geminiGenerateContent', 'response.chat.geminiGenerateContent.streamedUsage'], }); -export type GeminiServeEntry = C< - 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.gemini.wantsStream' - | 'request.chat.gemini' | 'serve.model' +export type GeminiGenerateContentServeEntry = C< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.geminiGenerateContent.wantsStream' + | 'request.chat.geminiGenerateContent' | 'serve.model' >; -export type GeminiServeExit = C< - 'response.chat.gemini.rendered' | 'response.chat.gemini.streamedUsage' +export type GeminiGenerateContentServeExit = C< + 'response.chat.geminiGenerateContent.rendered' | 'response.chat.geminiGenerateContent.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' >; -export const geminiServePipeline = (payload: GeminiPayload): Pipeline => - compose('geminiServe', [ +export const geminiGenerateContentServePipeline = (payload: GeminiGenerateContentPayload): Pipeline => + compose('geminiGenerateContentServe', [ emitGemini, writeSettlement( - handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini']), - handedUp => (handedUp as { 'response.chat.gemini.streamedUsage'?: unknown })['response.chat.gemini.streamedUsage'] !== null, + handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), + handedUp => (handedUp as { 'response.chat.geminiGenerateContent.streamedUsage'?: unknown })['response.chat.geminiGenerateContent.streamedUsage'] !== null, ), resolveChatCandidates(narrowing(payload)), failover({ - failed: handedUp => isFailure((handedUp as { 'response.chat.gemini'?: unknown })['response.chat.gemini']), + failed: handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), owns: [], }), - materializeAttempt('request.chat.gemini'), + materializeAttempt('request.chat.geminiGenerateContent'), stripUnsupportedPartFieldsFromGemini, stripUnsupportedToolsFromGemini, stripSafetySettingsFromGemini, suppressThoughtPartsFromGemini, requireGeminiTerminal, dialChatWire({ - source: 'request.chat.gemini', - needs: ['request.chat.gemini', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.gemini', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], - pick: endpoints => geminiTarget.pick(endpoints), - wire: geminiWireFor, + source: 'request.chat.geminiGenerateContent', + needs: ['request.chat.geminiGenerateContent', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.geminiGenerateContent', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => geminiGenerateContentTarget.pick(endpoints), + wire: geminiGenerateContentWireFor, }), ]); diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index 5287a8342c..3ac3386f60 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -25,15 +25,15 @@ import { isFailure, type AttemptSelector } from '../pipeline/facts.ts'; import { foldsExclusiveCacheTokens } from '../shared/telemetry/usage.ts'; import { defineStage, move, transform } from '@floway-dev/pipeline'; import type { - ChatCompletionsMessage, - ChatCompletionsPayload, - ChatCompletionsReasoningItem, - ChatCompletionsStreamEvent, -} from '@floway-dev/protocols/chat-completions'; + OpenAIChatCompletionsMessage, + OpenAIChatCompletionsPayload, + OpenAIChatCompletionsReasoningItem, + OpenAIChatCompletionsStreamEvent, +} from '@floway-dev/protocols/openai-chat-completions'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiContent, GeminiPayload, GeminiStreamEvent, GeminiToolGroup } from '@floway-dev/protocols/gemini'; -import type { MessagesPayload } from '@floway-dev/protocols/messages'; -import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesPayload, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { GeminiGenerateContentContent, GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent, GeminiGenerateContentToolGroup } from '@floway-dev/protocols/gemini-generate-content'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { CanonicalResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; /** Removing a field inside a value is the same move as removing a fact, and a value that * carried none of them comes back by identity so the rule costs nothing where it does not @@ -162,38 +162,38 @@ const attemptIdentity = (attempt: AttemptSelector): string => `${attempt.upstrea * that appears after the leading run — which is what "mid-conversation" means. The fold * itself is `roleRewriter`; what is here is the walk over this protocol's messages. */ export const applyRoleCompatibilityToChatCompletions = defineStage< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'applyRoleCompatibility', through: { request: { - needs: ['request.chat.chatCompletions', 'route.attempt'], + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], consumes: [], // Declared even though it will not always be written: a stage that only modifies a // field need not write on every run, and `provides \ needs` is the set that must be. - provides: ['request.chat.chatCompletions'], + provides: ['request.chat.openaiChatCompletions'], }, - response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => ({ request: facts => { const rewrite = rolesFor(facts['route.attempt'].flags); if (rewrite === null) return facts; // the free do-nothing path - const payload = facts['request.chat.chatCompletions']; + const payload = facts['request.chat.openaiChatCompletions']; const messages = rewriteChatCompletionsRoles(payload.messages, rewrite); // Written conditionally the whole way down, so a conversation this does not touch // comes back by identity and the layer costs what the rewrite actually changed. return messages === payload.messages ? facts - : { ...facts, 'request.chat.chatCompletions': move({ ...payload, messages }) }; + : { ...facts, 'request.chat.openaiChatCompletions': move({ ...payload, messages }) }; }, })), }); @@ -217,9 +217,9 @@ const rolesFor = (flags: readonly string[]): RoleRewrite | null => { : null; }; -type ChatCompletionsMessages = ChatCompletionsPayload['messages']; +type OpenAIChatCompletionsMessages = OpenAIChatCompletionsPayload['messages']; -const rewriteChatCompletionsRoles = (messages: ChatCompletionsMessages, rewrite: RoleRewrite): ChatCompletionsMessages => { +const rewriteChatCompletionsRoles = (messages: OpenAIChatCompletionsMessages, rewrite: RoleRewrite): OpenAIChatCompletionsMessages => { const nextRole = roleRewriter(rewrite); return mapKeepingIdentity(messages, message => { const role = nextRole(message.role); @@ -256,69 +256,69 @@ type RewrittenRole = 'system' | 'developer' | 'user'; * is the gateway's canonical form — putting it on the wire in a vendor's shape is the * vendor normalizer's job, which is why this runs above them. */ export const disableReasoningOnForcedToolChoiceForChatCompletions = defineStage< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'disableReasoningOnForcedToolChoice', through: { request: { - needs: ['request.chat.chatCompletions', 'route.attempt'], + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], consumes: [], - provides: ['request.chat.chatCompletions'], + provides: ['request.chat.openaiChatCompletions'], }, - response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { return facts; } - const payload = facts['request.chat.chatCompletions']; + const payload = facts['request.chat.openaiChatCompletions']; if (!isForcedToolChoice(payload.tool_choice)) return facts; - return { ...facts, 'request.chat.chatCompletions': move({ ...payload, reasoning_effort: 'none' }) }; + return { ...facts, 'request.chat.openaiChatCompletions': move({ ...payload, reasoning_effort: 'none' }) }; }, })), }); /** `required`, or a named function. `auto` and `none` leave the model free to reason. */ -const isForcedToolChoice = (choice: ChatCompletionsPayload['tool_choice']): boolean => +const isForcedToolChoice = (choice: OpenAIChatCompletionsPayload['tool_choice']): boolean => choice === 'required' || (typeof choice === 'object' && choice !== null); /** Drops a field the upstream would reject as an unknown argument. It runs above the vendor * normalizers so each of them sees the already-stripped canonical payload. */ export const stripPromptCacheKeyForChatCompletions = defineStage< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'stripPromptCacheKey', through: { request: { - needs: ['request.chat.chatCompletions', 'route.attempt'], + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], consumes: [], - provides: ['request.chat.chatCompletions'], + provides: ['request.chat.openaiChatCompletions'], }, - response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('strip-prompt-cache-key')) return facts; - const payload = facts['request.chat.chatCompletions']; + const payload = facts['request.chat.openaiChatCompletions']; const stripped = withoutKeys(payload, ['prompt_cache_key']); - return stripped === payload ? facts : { ...facts, 'request.chat.chatCompletions': move(stripped) }; + return stripped === payload ? facts : { ...facts, 'request.chat.openaiChatCompletions': move(stripped) }; }, })), }); @@ -330,7 +330,7 @@ export const stripPromptCacheKeyForChatCompletions = defineStage< * is on, and the gateway meters every stream from those frames — so what the client asked for * and what the upstream is asked for differ here, which is the one thing this rule exists to * say. Nothing downstream has to thread the client's original value through, because it is a - * fact of its own: `ingress.chat.chatCompletions.wantsUsageChunk` describes the request that + * fact of its own: `ingress.chat.openaiChatCompletions.wantsUsageChunk` describes the request that * arrived and survives this rewrite, and the edge reads it to decide who is shown the chunk. * * It belongs to the wire rather than to the source chain because it names a field of *this* @@ -342,34 +342,34 @@ export const stripPromptCacheKeyForChatCompletions = defineStage< * Reference: https://platform.openai.com/docs/api-reference/chat/create */ export const includeUsageStreamOptionsForChatCompletions = defineStage< - Chat<'request.chat.chatCompletions'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'includeUsageStreamOptions', through: { request: { - needs: ['request.chat.chatCompletions'], + needs: ['request.chat.openaiChatCompletions'], consumes: [], - provides: ['request.chat.chatCompletions'], + provides: ['request.chat.openaiChatCompletions'], }, - response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.chatCompletions'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => ({ request: facts => { - const payload = facts['request.chat.chatCompletions']; + const payload = facts['request.chat.openaiChatCompletions']; if (payload.stream_options?.include_usage === true) return facts; // Whatever else the client put on `stream_options` is its own and stays; only the one // field the gateway has an interest in is written. return { ...facts, - 'request.chat.chatCompletions': move({ + 'request.chat.openaiChatCompletions': move({ ...payload, stream_options: { ...payload.stream_options, include_usage: true }, }), @@ -393,27 +393,27 @@ export const includeUsageStreamOptionsForChatCompletions = defineStage< export const normalizeUsageForChatCompletions = defineStage< Record, Record, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'normalizeUsage', through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.chat.chatCompletions'], + needs: ['response.chat.openaiChatCompletions'], consumes: [], - provides: ['response.chat.chatCompletions'], + provides: ['response.chat.openaiChatCompletions'], }, }, execute: transform< Record, Record, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => ({ response: facts => { - const answer = answerWithFrames(facts['response.chat.chatCompletions'], withUsageOnItsOwnCarrier); - return answer === null ? facts : { ...facts, 'response.chat.chatCompletions': move(answer) }; + const answer = answerWithFrames(facts['response.chat.openaiChatCompletions'], withUsageOnItsOwnCarrier); + return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; }, })), }); @@ -421,8 +421,8 @@ export const normalizeUsageForChatCompletions = defineStage< /** One chunk in, two out — and only for the chunk that carried both, which is why this is not * a one-for-one event rewrite. */ const withUsageOnItsOwnCarrier = async function* ( - frames: AsyncIterable>, -): AsyncGenerator> { + frames: AsyncIterable>, +): AsyncGenerator> { for await (const frame of frames) { if (frame.type !== 'event') { yield frame; @@ -462,23 +462,23 @@ const withUsageOnItsOwnCarrier = async function* ( export const normalizeExclusiveCachedTokensForChatCompletions = defineStage< Chat<'route.attempt'>, Chat<'route.attempt'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'normalizeExclusiveCachedTokens', through: { request: { needs: ['route.attempt'], consumes: [], provides: [] }, response: { - needs: ['response.chat.chatCompletions'], + needs: ['response.chat.openaiChatCompletions'], consumes: [], - provides: ['response.chat.chatCompletions'], + provides: ['response.chat.openaiChatCompletions'], }, }, execute: transform< Chat<'route.attempt'>, Chat<'route.attempt'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => { // Which attempt this is gets read on the way down and spoken about on the way back, which // is the order `transform` runs the two halves in. @@ -491,11 +491,11 @@ export const normalizeExclusiveCachedTokensForChatCompletions = defineStage< response: facts => { const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); const identity = attemptIdentity(attempt); - const answer = answerWithFrames( - facts['response.chat.chatCompletions'], + const answer = answerWithFrames( + facts['response.chat.openaiChatCompletions'], frames => rewritingEvents(frames, chunk => foldChatCompletionsUsage(chunk, declaredExclusive, identity)), ); - return answer === null ? facts : { ...facts, 'response.chat.chatCompletions': move(answer) }; + return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; }, }; }), @@ -511,14 +511,14 @@ const CHAT_COMPLETIONS_CACHE_BUCKETS: CacheBucketNames = { }; const foldChatCompletionsUsage = ( - chunk: ChatCompletionsStreamEvent, + chunk: OpenAIChatCompletionsStreamEvent, declaredExclusive: boolean, identity: string, -): ChatCompletionsStreamEvent => { +): OpenAIChatCompletionsStreamEvent => { const usage = asJsonObject(chunk.usage); if (usage === null) return chunk; const folded = withCacheBucketsFolded(usage, CHAT_COMPLETIONS_CACHE_BUCKETS, declaredExclusive, identity); - return folded === usage ? chunk : { ...chunk, usage: folded as unknown as ChatCompletionsStreamEvent['usage'] }; + return folded === usage ? chunk : { ...chunk, usage: folded as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; }; // ── Chat Completions vendor dialects ────────────────────────────────────────────────────── @@ -549,29 +549,29 @@ const foldChatCompletionsUsage = ( * - https://api-docs.deepseek.com/quick_start/agent_integrations/oh_my_pi */ export const vendorDeepSeekNormalizeForChatCompletions = defineStage< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'vendorDeepSeekNormalize', through: { request: { - needs: ['request.chat.chatCompletions', 'route.attempt'], + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], consumes: [], - provides: ['request.chat.chatCompletions'], + provides: ['request.chat.openaiChatCompletions'], }, response: { - needs: ['response.chat.chatCompletions'], + needs: ['response.chat.openaiChatCompletions'], consumes: [], - provides: ['response.chat.chatCompletions'], + provides: ['response.chat.openaiChatCompletions'], }, }, execute: transform< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => { // Whether this vendor's dialect applies at all is read on the way down and acted on in // both directions, which is the order `transform` runs the two halves in. @@ -580,25 +580,25 @@ export const vendorDeepSeekNormalizeForChatCompletions = defineStage< request: facts => { enabled = facts['route.attempt'].flags.includes('vendor-deepseek'); if (!enabled) return facts; - const payload = facts['request.chat.chatCompletions']; + const payload = facts['request.chat.openaiChatCompletions']; const normalized = deepSeekOutbound(payload); - return normalized === payload ? facts : { ...facts, 'request.chat.chatCompletions': move(normalized) }; + return normalized === payload ? facts : { ...facts, 'request.chat.openaiChatCompletions': move(normalized) }; }, response: facts => { if (!enabled) return facts; - const answer = answerWithFrames( - facts['response.chat.chatCompletions'], + const answer = answerWithFrames( + facts['response.chat.openaiChatCompletions'], frames => rewritingEvents(frames, chunk => deepSeekInboundUsage(deepSeekInboundDeltas(chunk))), ); - return answer === null ? facts : { ...facts, 'response.chat.chatCompletions': move(answer) }; + return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; }, }; }), }); -const deepSeekOutbound = (payload: ChatCompletionsPayload): ChatCompletionsPayload => { +const deepSeekOutbound = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsPayload => { const withThinking = payload.reasoning_effort === 'none' - ? { ...withoutKeys(payload, ['reasoning_effort']), thinking: { type: 'disabled' } } satisfies ChatCompletionsPayloadWithDeepSeekThinking + ? { ...withoutKeys(payload, ['reasoning_effort']), thinking: { type: 'disabled' } } satisfies OpenAIChatCompletionsPayloadWithDeepSeekThinking : payload; const withResponseFormat = withThinking.response_format?.type === 'json_schema' ? { ...withThinking, response_format: { type: 'json_object' } } @@ -610,27 +610,27 @@ const deepSeekOutbound = (payload: ChatCompletionsPayload): ChatCompletionsPaylo /** The reasoning DeepSeek reads, and only that: `reasoning_opaque` is the OpenAI-canonical * signature for cross-turn replay and DeepSeek does not accept it, so it goes with the two * fields that are projected onto `reasoning_content`. */ -const deepSeekAssistantReasoning = (message: ChatCompletionsMessage): ChatCompletionsMessage => { +const deepSeekAssistantReasoning = (message: OpenAIChatCompletionsMessage): OpenAIChatCompletionsMessage => { const stripped = withoutKeys(message, ['reasoning_text', 'reasoning_opaque', 'reasoning_items']); if (stripped === message) return message; const text = typeof message.reasoning_text === 'string' ? message.reasoning_text : deepSeekReasoningFromItems(message.reasoning_items); if (text === undefined) return stripped; - const projected: ChatCompletionsMessageWithDeepSeekReasoning = { ...stripped, reasoning_content: text }; + const projected: OpenAIChatCompletionsMessageWithDeepSeekReasoning = { ...stripped, reasoning_content: text }; return projected; }; /** The newer OpenAI shape carries reasoning as summary items; DeepSeek documents only the * scalar, so what summaries there are become it. */ -const deepSeekReasoningFromItems = (items: ChatCompletionsReasoningItem[] | null | undefined): string | undefined => { +const deepSeekReasoningFromItems = (items: OpenAIChatCompletionsReasoningItem[] | null | undefined): string | undefined => { const parts = items?.flatMap(item => item.summary?.map(summary => summary.text) ?? []) ?? []; return parts.length > 0 ? parts.join('') : undefined; }; -const deepSeekInboundDeltas = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { +const deepSeekInboundDeltas = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { const choices = mapKeepingIdentity(chunk.choices, choice => { - const delta = choice.delta as ChatCompletionsDeltaWithDeepSeekReasoning; + const delta = choice.delta as OpenAIChatCompletionsDeltaWithDeepSeekReasoning; if (typeof delta.reasoning_content !== 'string') return choice; const stripped = withoutKeys(delta, ['reasoning_content']); return { @@ -648,7 +648,7 @@ const deepSeekInboundDeltas = (chunk: ChatCompletionsStreamEvent): ChatCompletio * https://api-docs.deepseek.com/guides/kv_cache */ const DEEPSEEK_CACHE_FIELDS = ['prompt_cache_hit_tokens', 'prompt_cache_miss_tokens'] as const; -const deepSeekInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { +const deepSeekInboundUsage = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { const usage = asJsonObject(chunk.usage); if (usage === null) return chunk; const stripped = withoutKeys(usage, DEEPSEEK_CACHE_FIELDS); @@ -657,7 +657,7 @@ const deepSeekInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletion const next: JsonObject = hit == null ? stripped : { ...stripped, prompt_tokens_details: { ...(asJsonObject(usage.prompt_tokens_details) ?? {}), cached_tokens: hit } }; - return { ...chunk, usage: next as unknown as ChatCompletionsStreamEvent['usage'] }; + return { ...chunk, usage: next as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; }; /** Qwen says "no reasoning" with a top-level `enable_thinking: false` rather than with the @@ -665,35 +665,35 @@ const deepSeekInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletion * there is nothing to do on the way back. * https://www.alibabacloud.com/help/en/model-studio/deep-thinking */ export const vendorQwenNormalizeForChatCompletions = defineStage< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'vendorQwenNormalize', through: { request: { - needs: ['request.chat.chatCompletions', 'route.attempt'], + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], consumes: [], - provides: ['request.chat.chatCompletions'], + provides: ['request.chat.openaiChatCompletions'], }, - response: { needs: ['response.chat.chatCompletions'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.chatCompletions' | 'route.attempt'>, - Chat<'request.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('vendor-qwen')) return facts; - const payload = facts['request.chat.chatCompletions']; + const payload = facts['request.chat.openaiChatCompletions']; if (payload.reasoning_effort !== 'none') return facts; - const normalized: ChatCompletionsPayloadWithQwenThinking = { + const normalized: OpenAIChatCompletionsPayloadWithQwenThinking = { ...withoutKeys(payload, ['reasoning_effort']), enable_thinking: false, }; - return { ...facts, 'request.chat.chatCompletions': move(normalized) }; + return { ...facts, 'request.chat.openaiChatCompletions': move(normalized) }; }, })), }); @@ -705,23 +705,23 @@ export const vendorQwenNormalizeForChatCompletions = defineStage< export const vendorKimiNormalizeForChatCompletions = defineStage< Chat<'route.attempt'>, Chat<'route.attempt'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >({ name: 'vendorKimiNormalize', through: { request: { needs: ['route.attempt'], consumes: [], provides: [] }, response: { - needs: ['response.chat.chatCompletions'], + needs: ['response.chat.openaiChatCompletions'], consumes: [], - provides: ['response.chat.chatCompletions'], + provides: ['response.chat.openaiChatCompletions'], }, }, execute: transform< Chat<'route.attempt'>, Chat<'route.attempt'>, - Chat<'response.chat.chatCompletions'>, - Chat<'response.chat.chatCompletions'> + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> >(() => { let enabled = false; return { @@ -731,17 +731,17 @@ export const vendorKimiNormalizeForChatCompletions = defineStage< }, response: facts => { if (!enabled) return facts; - const answer = answerWithFrames( - facts['response.chat.chatCompletions'], + const answer = answerWithFrames( + facts['response.chat.openaiChatCompletions'], frames => rewritingEvents(frames, kimiInboundUsage), ); - return answer === null ? facts : { ...facts, 'response.chat.chatCompletions': move(answer) }; + return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; }, }; }), }); -const kimiInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStreamEvent => { +const kimiInboundUsage = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { const usage = asJsonObject(chunk.usage); if (usage === null) return chunk; const cached = readJsonNumber(usage.cached_tokens); @@ -750,17 +750,17 @@ const kimiInboundUsage = (chunk: ChatCompletionsStreamEvent): ChatCompletionsStr ...withoutKeys(usage, ['cached_tokens']), prompt_tokens_details: { ...(asJsonObject(usage.prompt_tokens_details) ?? {}), cached_tokens: cached }, }; - return { ...chunk, usage: next as unknown as ChatCompletionsStreamEvent['usage'] }; + return { ...chunk, usage: next as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; }; /** None of the three fields is a Chat Completions field, so each is declared beside the vendor * that reads it rather than widening the protocol's own types. */ -type ChatCompletionsPayloadWithDeepSeekThinking = Omit & { thinking: { type: string } }; -type ChatCompletionsPayloadWithQwenThinking = Omit & { enable_thinking: false }; -type ChatCompletionsMessageWithDeepSeekReasoning = - Omit & { reasoning_content: string }; -type ChatCompletionsDeltaWithDeepSeekReasoning = - ChatCompletionsStreamEvent['choices'][number]['delta'] & { reasoning_content?: unknown }; +type OpenAIChatCompletionsPayloadWithDeepSeekThinking = Omit & { thinking: { type: string } }; +type OpenAIChatCompletionsPayloadWithQwenThinking = Omit & { enable_thinking: false }; +type OpenAIChatCompletionsMessageWithDeepSeekReasoning = + Omit & { reasoning_content: string }; +type OpenAIChatCompletionsDeltaWithDeepSeekReasoning = + OpenAIChatCompletionsStreamEvent['choices'][number]['delta'] & { reasoning_content?: unknown }; // ── Messages ────────────────────────────────────────────────────────────────────────────── @@ -773,34 +773,34 @@ type ChatCompletionsDeltaWithDeepSeekReasoning = * cross here, and no developer role to trade with. */ export const applyRoleCompatibilityToMessages = defineStage< - Chat<'request.chat.messages' | 'route.attempt'>, - Chat<'request.chat.messages'>, - Chat<'response.chat.messages'>, - Chat<'response.chat.messages'> + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> >({ name: 'applyRoleCompatibility', through: { request: { - needs: ['request.chat.messages', 'route.attempt'], + needs: ['request.chat.anthropicMessages', 'route.attempt'], consumes: [], - provides: ['request.chat.messages'], + provides: ['request.chat.anthropicMessages'], }, - response: { needs: ['response.chat.messages'], consumes: [], provides: [] }, + response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.messages' | 'route.attempt'>, - Chat<'request.chat.messages'>, - Chat<'response.chat.messages'>, - Chat<'response.chat.messages'> + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('rewrite-mid-conv-system-to-user')) return facts; - const payload = facts['request.chat.messages']; + const payload = facts['request.chat.anthropicMessages']; const messages = mapKeepingIdentity(payload.messages, message => message.role === 'system' ? { role: 'user' as const, content: message.content } : message); return messages === payload.messages ? facts - : { ...facts, 'request.chat.messages': move({ ...payload, messages }) }; + : { ...facts, 'request.chat.anthropicMessages': move({ ...payload, messages }) }; }, })), }); @@ -809,34 +809,34 @@ export const applyRoleCompatibilityToMessages = defineStage< * natively. `tool` and `any` are the forced choices; `auto` and `none` leave the model free * to reason. */ export const disableReasoningOnForcedToolChoiceForMessages = defineStage< - Chat<'request.chat.messages' | 'route.attempt'>, - Chat<'request.chat.messages'>, - Chat<'response.chat.messages'>, - Chat<'response.chat.messages'> + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> >({ name: 'disableReasoningOnForcedToolChoice', through: { request: { - needs: ['request.chat.messages', 'route.attempt'], + needs: ['request.chat.anthropicMessages', 'route.attempt'], consumes: [], - provides: ['request.chat.messages'], + provides: ['request.chat.anthropicMessages'], }, - response: { needs: ['response.chat.messages'], consumes: [], provides: [] }, + response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.messages' | 'route.attempt'>, - Chat<'request.chat.messages'>, - Chat<'response.chat.messages'>, - Chat<'response.chat.messages'> + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { return facts; } - const payload = facts['request.chat.messages']; + const payload = facts['request.chat.anthropicMessages']; const choice = payload.tool_choice?.type; if (choice !== 'tool' && choice !== 'any') return facts; - return { ...facts, 'request.chat.messages': move(withMessagesReasoningDisabled(payload)) }; + return { ...facts, 'request.chat.anthropicMessages': move(withMessagesReasoningDisabled(payload)) }; }, })), }); @@ -845,9 +845,9 @@ export const disableReasoningOnForcedToolChoiceForMessages = defineStage< * with structured output on these upstreams, and it is thinking it does not compose with, so * `output_config.format` has to survive. An `output_config` that held nothing but the effort * is nothing at all once the effort is gone, so it goes rather than riding on empty. */ -const withMessagesReasoningDisabled = (payload: MessagesPayload): MessagesPayload => { +const withMessagesReasoningDisabled = (payload: AnthropicMessagesPayload): AnthropicMessagesPayload => { const { output_config, ...rest } = payload; - const next: MessagesPayload = { ...rest, thinking: { type: 'disabled' as const } }; + const next: AnthropicMessagesPayload = { ...rest, thinking: { type: 'disabled' as const } }; if (output_config !== undefined) { const remaining = withoutKeys(output_config, ['effort']); if (Object.keys(remaining).length > 0) next.output_config = remaining; @@ -874,35 +874,35 @@ const withMessagesReasoningDisabled = (payload: MessagesPayload): MessagesPayloa * *more* of these fields on its first-party path, so it reads as a client-attribution signal. */ export const stripBillingAttributionFromMessages = defineStage< - Chat<'request.chat.messages' | 'route.attempt'>, - Chat<'request.chat.messages'>, - Chat<'response.chat.messages'>, - Chat<'response.chat.messages'> + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> >({ name: 'stripBillingAttribution', through: { request: { - needs: ['request.chat.messages', 'route.attempt'], + needs: ['request.chat.anthropicMessages', 'route.attempt'], consumes: [], - provides: ['request.chat.messages'], + provides: ['request.chat.anthropicMessages'], }, - response: { needs: ['response.chat.messages'], consumes: [], provides: [] }, + response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.messages' | 'route.attempt'>, - Chat<'request.chat.messages'>, - Chat<'response.chat.messages'>, - Chat<'response.chat.messages'> + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('strip-billing-attribution')) return facts; - const payload = facts['request.chat.messages']; + const payload = facts['request.chat.anthropicMessages']; const system = scrubbedSystemPrompt(payload.system); if (system === payload.system) return facts; const withoutSystem = withoutKeys(payload, ['system']); return { ...facts, - 'request.chat.messages': move(system === undefined ? withoutSystem : { ...withoutSystem, system }), + 'request.chat.anthropicMessages': move(system === undefined ? withoutSystem : { ...withoutSystem, system }), }; }, })), @@ -921,7 +921,7 @@ const scrubText = (text: string): string => /** What is left of the system prompt once the block is out of it: the value that came in when * there was nothing to scrub, and `undefined` when nothing at all is left — an empty system * prompt is not a system prompt, so the field goes rather than riding on empty. */ -const scrubbedSystemPrompt = (system: MessagesPayload['system']): MessagesPayload['system'] => { +const scrubbedSystemPrompt = (system: AnthropicMessagesPayload['system']): AnthropicMessagesPayload['system'] => { if (system === undefined) return undefined; if (typeof system === 'string') { const scrubbed = scrubText(system); @@ -952,26 +952,26 @@ const scrubbedSystemPrompt = (system: MessagesPayload['system']): MessagesPayloa * at source and every target sees translatable parts. A part left holding nothing goes with * them: it is not a part any more. */ export const stripUnsupportedPartFieldsFromGemini = defineStage< - Chat<'request.chat.gemini'>, - Chat<'request.chat.gemini'>, - Chat<'response.chat.gemini'>, - Chat<'response.chat.gemini'> + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> >({ name: 'stripUnsupportedPartFields', through: { - request: { needs: ['request.chat.gemini'], consumes: [], provides: ['request.chat.gemini'] }, - response: { needs: ['response.chat.gemini'], consumes: [], provides: [] }, + request: { needs: ['request.chat.geminiGenerateContent'], consumes: [], provides: ['request.chat.geminiGenerateContent'] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.gemini'>, - Chat<'request.chat.gemini'>, - Chat<'response.chat.gemini'>, - Chat<'response.chat.gemini'> + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> >(() => ({ request: facts => { - const payload = facts['request.chat.gemini']; + const payload = facts['request.chat.geminiGenerateContent']; const stripped = withUnsupportedPartFieldsStripped(payload); - return stripped === payload ? facts : { ...facts, 'request.chat.gemini': move(stripped) }; + return stripped === payload ? facts : { ...facts, 'request.chat.geminiGenerateContent': move(stripped) }; }, })), }); @@ -984,7 +984,7 @@ export const stripUnsupportedPartFieldsFromGemini = defineStage< * https://ai.google.dev/gemini-api/docs/code-execution */ const UNSUPPORTED_GEMINI_PART_FIELDS = ['fileData', 'executableCode', 'codeExecutionResult'] as const; -const withUnsupportedPartFieldsStripped = (payload: GeminiPayload): GeminiPayload => { +const withUnsupportedPartFieldsStripped = (payload: GeminiGenerateContentPayload): GeminiGenerateContentPayload => { const contents = payload.contents === undefined ? undefined : mapKeepingIdentity(payload.contents, stripContentParts); @@ -999,7 +999,7 @@ const withUnsupportedPartFieldsStripped = (payload: GeminiPayload): GeminiPayloa }; }; -const stripContentParts = (content: GeminiContent): GeminiContent => { +const stripContentParts = (content: GeminiGenerateContentContent): GeminiGenerateContentContent => { let changed = false; const parts = content.parts.flatMap(part => { const stripped = withoutKeys(part, UNSUPPORTED_GEMINI_PART_FIELDS); @@ -1015,26 +1015,26 @@ const stripContentParts = (content: GeminiContent): GeminiContent => { * emitter offered an empty group would be offered a tool that does nothing. `tools` itself * goes when no group survived: an empty tool list is a different request from no tools. */ export const stripUnsupportedToolsFromGemini = defineStage< - Chat<'request.chat.gemini'>, - Chat<'request.chat.gemini'>, - Chat<'response.chat.gemini'>, - Chat<'response.chat.gemini'> + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> >({ name: 'stripUnsupportedTools', through: { - request: { needs: ['request.chat.gemini'], consumes: [], provides: ['request.chat.gemini'] }, - response: { needs: ['response.chat.gemini'], consumes: [], provides: [] }, + request: { needs: ['request.chat.geminiGenerateContent'], consumes: [], provides: ['request.chat.geminiGenerateContent'] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.gemini'>, - Chat<'request.chat.gemini'>, - Chat<'response.chat.gemini'>, - Chat<'response.chat.gemini'> + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> >(() => ({ request: facts => { - const payload = facts['request.chat.gemini']; + const payload = facts['request.chat.geminiGenerateContent']; const stripped = withUnsupportedToolsStripped(payload); - return stripped === payload ? facts : { ...facts, 'request.chat.gemini': move(stripped) }; + return stripped === payload ? facts : { ...facts, 'request.chat.geminiGenerateContent': move(stripped) }; }, })), }); @@ -1057,11 +1057,11 @@ const UNSUPPORTED_GEMINI_TOOL_CAPABILITIES = [ 'googleMaps', ] as const; -const withUnsupportedToolsStripped = (payload: GeminiPayload): GeminiPayload => { +const withUnsupportedToolsStripped = (payload: GeminiGenerateContentPayload): GeminiGenerateContentPayload => { const { tools } = payload; if (tools === undefined) return payload; let changed = false; - const kept: GeminiToolGroup[] = []; + const kept: GeminiGenerateContentToolGroup[] = []; for (const tool of tools) { const stripped = withoutKeys(tool, UNSUPPORTED_GEMINI_TOOL_CAPABILITIES); if (stripped !== tool) changed = true; @@ -1076,26 +1076,26 @@ const withUnsupportedToolsStripped = (payload: GeminiPayload): GeminiPayload => * path, so they go rather than have us pretend to enforce a policy we cannot honor * end to end. */ export const stripSafetySettingsFromGemini = defineStage< - Chat<'request.chat.gemini'>, - Chat<'request.chat.gemini'>, - Chat<'response.chat.gemini'>, - Chat<'response.chat.gemini'> + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> >({ name: 'stripSafetySettings', through: { - request: { needs: ['request.chat.gemini'], consumes: [], provides: ['request.chat.gemini'] }, - response: { needs: ['response.chat.gemini'], consumes: [], provides: [] }, + request: { needs: ['request.chat.geminiGenerateContent'], consumes: [], provides: ['request.chat.geminiGenerateContent'] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.gemini'>, - Chat<'request.chat.gemini'>, - Chat<'response.chat.gemini'>, - Chat<'response.chat.gemini'> + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> >(() => ({ request: facts => { - const payload = facts['request.chat.gemini']; + const payload = facts['request.chat.geminiGenerateContent']; const stripped = withoutKeys(payload, ['safetySettings']); - return stripped === payload ? facts : { ...facts, 'request.chat.gemini': move(stripped) }; + return stripped === payload ? facts : { ...facts, 'request.chat.geminiGenerateContent': move(stripped) }; }, })), }); @@ -1109,40 +1109,40 @@ export const stripSafetySettingsFromGemini = defineStage< * `needs` can only name what the ending provides, which is the answer and not the turn. */ export const suppressThoughtPartsFromGemini = defineStage< - Chat<'request.chat.gemini'>, - Chat<'request.chat.gemini'>, - Chat<'response.chat.gemini'>, - Chat<'response.chat.gemini'> + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> >({ name: 'suppressThoughtParts', through: { - request: { needs: ['request.chat.gemini'], consumes: [], provides: [] }, - response: { needs: ['response.chat.gemini'], consumes: [], provides: ['response.chat.gemini'] }, + request: { needs: ['request.chat.geminiGenerateContent'], consumes: [], provides: [] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: ['response.chat.geminiGenerateContent'] }, }, execute: transform< - Chat<'request.chat.gemini'>, - Chat<'request.chat.gemini'>, - Chat<'response.chat.gemini'>, - Chat<'response.chat.gemini'> + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> >(() => { // Assigned on the way down and read on the way back, which is the order `transform` runs // the two halves in. let includeThoughts!: boolean; return { request: facts => { - includeThoughts = facts['request.chat.gemini'].generationConfig?.thinkingConfig?.includeThoughts === true; + includeThoughts = facts['request.chat.geminiGenerateContent'].generationConfig?.thinkingConfig?.includeThoughts === true; return facts; }, response: facts => { if (includeThoughts) return facts; - const answer = facts['response.chat.gemini']; + const answer = facts['response.chat.geminiGenerateContent']; // A refusal and a collected body have no thought parts to hide, and telling the three // apart is reading a value rather than dispatching on a declaration. if (isFailure(answer) || answer.kind !== 'stream') return facts; - const frames = answer.frames as AsyncIterable>; + const frames = answer.frames as AsyncIterable>; return { ...facts, - 'response.chat.gemini': move({ + 'response.chat.geminiGenerateContent': move({ kind: 'stream' as const, frames: { [Symbol.asyncIterator]: () => withoutThoughtParts(frames) }, }), @@ -1157,8 +1157,8 @@ export const suppressThoughtPartsFromGemini = defineStage< * both is what keeps a stream of pure thought from reaching the client as a stream of * empties. */ const withoutThoughtParts = async function* ( - frames: AsyncIterable>, -): AsyncGenerator> { + frames: AsyncIterable>, +): AsyncGenerator> { for await (const frame of frames) { if (frame.type !== 'event' || 'error' in frame.event) { yield frame; @@ -1171,7 +1171,7 @@ const withoutThoughtParts = async function* ( return [{ ...candidate, content: { ...candidate.content, parts } }]; }); - const event: GeminiStreamEvent = { + const event: GeminiGenerateContentStreamEvent = { ...frame.event, ...(candidates === undefined ? {} : { candidates }), }; @@ -1179,7 +1179,7 @@ const withoutThoughtParts = async function* ( } }; -const hasGeminiEventPayload = (event: GeminiStreamEvent): boolean => { +const hasGeminiEventPayload = (event: GeminiGenerateContentStreamEvent): boolean => { if ('error' in event) return true; return (event.candidates?.length ?? 0) > 0 || event.usageMetadata !== undefined @@ -1194,34 +1194,34 @@ const hasGeminiEventPayload = (event: GeminiStreamEvent): boolean => { * an item that carries none still crosses the leading system run — a reasoning item between * two system messages is what makes the second one mid-conversation. */ export const applyRoleCompatibilityToResponses = defineStage< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >({ name: 'applyRoleCompatibility', through: { request: { - needs: ['request.chat.responses', 'route.attempt'], + needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], - provides: ['request.chat.responses'], + provides: ['request.chat.openaiResponses'], }, - response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >(() => ({ request: facts => { const rewrite = rolesFor(facts['route.attempt'].flags); if (rewrite === null) return facts; // the free do-nothing path // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to, which is the one a wire takes. - const payload = facts['request.chat.responses'] as CanonicalResponsesPayload; + const payload = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; const nextRole = roleRewriter(rewrite); - const input = mapKeepingIdentity(payload.input, (item): ResponsesInputItem => { + const input = mapKeepingIdentity(payload.input, (item): OpenAIResponsesInputItem => { if (item.type !== 'message') { nextRole(undefined); return item; @@ -1231,7 +1231,7 @@ export const applyRoleCompatibilityToResponses = defineStage< }); return input === payload.input ? facts - : { ...facts, 'request.chat.responses': move({ ...payload, input }) }; + : { ...facts, 'request.chat.openaiResponses': move({ ...payload, input }) }; }, })), }); @@ -1242,38 +1242,38 @@ export const applyRoleCompatibilityToResponses = defineStage< * the sentinel is the gateway's canonical form, which is why this runs above the vendor * normalizers that put it on the wire in a vendor's shape. */ export const disableReasoningOnForcedToolChoiceForResponses = defineStage< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >({ name: 'disableReasoningOnForcedToolChoice', through: { request: { - needs: ['request.chat.responses', 'route.attempt'], + needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], - provides: ['request.chat.responses'], + provides: ['request.chat.openaiResponses'], }, - response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { return facts; } - const payload = facts['request.chat.responses']; + const payload = facts['request.chat.openaiResponses']; if (!isForcedResponsesToolChoice(payload.tool_choice)) return facts; - return { ...facts, 'request.chat.responses': move({ ...payload, reasoning: { effort: 'none' } }) }; + return { ...facts, 'request.chat.openaiResponses': move({ ...payload, reasoning: { effort: 'none' } }) }; }, })), }); -const isForcedResponsesToolChoice = (choice: ResponsesPayload['tool_choice']): boolean => { +const isForcedResponsesToolChoice = (choice: OpenAIResponsesPayload['tool_choice']): boolean => { if (choice === undefined || choice === null) return false; if (typeof choice === 'string') return choice === 'required'; return true; @@ -1282,31 +1282,31 @@ const isForcedResponsesToolChoice = (choice: ResponsesPayload['tool_choice']): b /** Drops a field the upstream would reject as an unknown argument. It runs above the vendor * normalizers so each of them sees the already-stripped canonical payload. */ export const stripPromptCacheKeyForResponses = defineStage< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >({ name: 'stripPromptCacheKey', through: { request: { - needs: ['request.chat.responses', 'route.attempt'], + needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], - provides: ['request.chat.responses'], + provides: ['request.chat.openaiResponses'], }, - response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('strip-prompt-cache-key')) return facts; - const payload = facts['request.chat.responses']; + const payload = facts['request.chat.openaiResponses']; const stripped = withoutKeys(payload, ['prompt_cache_key']); - return stripped === payload ? facts : { ...facts, 'request.chat.responses': move(stripped) }; + return stripped === payload ? facts : { ...facts, 'request.chat.openaiResponses': move(stripped) }; }, })), }); @@ -1326,19 +1326,19 @@ export const stripPromptCacheKeyForResponses = defineStage< export const normalizeExclusiveCachedTokensForResponses = defineStage< Chat<'route.attempt'>, Chat<'route.attempt'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >({ name: 'normalizeExclusiveCachedTokens', through: { request: { needs: ['route.attempt'], consumes: [], provides: [] }, - response: { needs: ['response.chat.responses'], consumes: [], provides: ['response.chat.responses'] }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: ['response.chat.openaiResponses'] }, }, execute: transform< Chat<'route.attempt'>, Chat<'route.attempt'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >(() => { // Which attempt this is gets read on the way down and spoken about on the way back, which // is the order `transform` runs the two halves in. @@ -1351,11 +1351,11 @@ export const normalizeExclusiveCachedTokensForResponses = defineStage< response: facts => { const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); const identity = attemptIdentity(attempt); - const answer = answerWithFrames( - facts['response.chat.responses'], + const answer = answerWithFrames( + facts['response.chat.openaiResponses'], frames => rewritingEvents(frames, event => foldResponsesUsage(event, declaredExclusive, identity)), ); - return answer === null ? facts : { ...facts, 'response.chat.responses': move(answer) }; + return answer === null ? facts : { ...facts, 'response.chat.openaiResponses': move(answer) }; }, }; }), @@ -1371,17 +1371,17 @@ const RESPONSES_CACHE_BUCKETS: CacheBucketNames = { }; const foldResponsesUsage = ( - event: ResponsesStreamEvent, + event: OpenAIResponsesStreamEvent, declaredExclusive: boolean, identity: string, -): ResponsesStreamEvent => { +): OpenAIResponsesStreamEvent => { if (!('response' in event)) return event; const response = asJsonObject(event.response); const usage = asJsonObject(response?.usage); if (response === null || usage === null) return event; const folded = withCacheBucketsFolded(usage, RESPONSES_CACHE_BUCKETS, declaredExclusive, identity); if (folded === usage) return event; - return { ...event, response: { ...response, usage: folded } as JsonObject } as unknown as ResponsesStreamEvent; + return { ...event, response: { ...response, usage: folded } as JsonObject } as unknown as OpenAIResponsesStreamEvent; }; /** DeepSeek says "no reasoning" with a top-level `thinking: { type: 'disabled' }` rather than @@ -1389,33 +1389,33 @@ const foldResponsesUsage = ( * this chain's rewrites, because a vendor normalizer has the final say on the outbound body. * https://api-docs.deepseek.com/zh-cn/guides/thinking_mode */ export const vendorDeepSeekNormalizeForResponses = defineStage< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >({ name: 'vendorDeepSeekNormalize', through: { request: { - needs: ['request.chat.responses', 'route.attempt'], + needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], - provides: ['request.chat.responses'], + provides: ['request.chat.openaiResponses'], }, - response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('vendor-deepseek')) return facts; - const payload = facts['request.chat.responses']; + const payload = facts['request.chat.openaiResponses']; if (payload.reasoning?.effort !== 'none') return facts; const { reasoning, ...rest } = payload; - const normalized: ResponsesPayloadWithDeepSeekThinking = { ...rest, thinking: { type: 'disabled' } }; - return { ...facts, 'request.chat.responses': move(normalized) }; + const normalized: OpenAIResponsesPayloadWithDeepSeekThinking = { ...rest, thinking: { type: 'disabled' } }; + return { ...facts, 'request.chat.openaiResponses': move(normalized) }; }, })), }); @@ -1423,38 +1423,38 @@ export const vendorDeepSeekNormalizeForResponses = defineStage< /** Qwen says the same thing with a top-level `enable_thinking: false`. * https://www.alibabacloud.com/help/en/model-studio/deep-thinking */ export const vendorQwenNormalizeForResponses = defineStage< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >({ name: 'vendorQwenNormalize', through: { request: { - needs: ['request.chat.responses', 'route.attempt'], + needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], - provides: ['request.chat.responses'], + provides: ['request.chat.openaiResponses'], }, - response: { needs: ['response.chat.responses'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, }, execute: transform< - Chat<'request.chat.responses' | 'route.attempt'>, - Chat<'request.chat.responses'>, - Chat<'response.chat.responses'>, - Chat<'response.chat.responses'> + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> >(() => ({ request: facts => { if (!facts['route.attempt'].flags.includes('vendor-qwen')) return facts; - const payload = facts['request.chat.responses']; + const payload = facts['request.chat.openaiResponses']; if (payload.reasoning?.effort !== 'none') return facts; const { reasoning, ...rest } = payload; - const normalized: ResponsesPayloadWithQwenThinking = { ...rest, enable_thinking: false }; - return { ...facts, 'request.chat.responses': move(normalized) }; + const normalized: OpenAIResponsesPayloadWithQwenThinking = { ...rest, enable_thinking: false }; + return { ...facts, 'request.chat.openaiResponses': move(normalized) }; }, })), }); /** Neither field is a Responses field, so each is declared beside the vendor that reads it * rather than widening the protocol's own request type. */ -type ResponsesPayloadWithDeepSeekThinking = Omit & { thinking: { type: 'disabled' } }; -type ResponsesPayloadWithQwenThinking = Omit & { enable_thinking: false }; +type OpenAIResponsesPayloadWithDeepSeekThinking = Omit & { thinking: { type: 'disabled' } }; +type OpenAIResponsesPayloadWithQwenThinking = Omit & { enable_thinking: false }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts index 9d965bb2c7..d111f36f94 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts @@ -1,6 +1,6 @@ // POST /v1/chat/completions, served through the pipeline. // -// The handler is a prologue and an epilogue around `chatCompletionsServePipeline`: read what +// The handler is a prologue and an epilogue around `openaiChatCompletionsServePipeline`: read what // the client sent, hand it over, and turn what the run answered with into a response. // Everything between is stages. // @@ -9,26 +9,26 @@ // the run has to be opened knowing, and whether it asked to be shown the usage chunk that // metering asks the upstream for on every streaming turn. -import { chatCompletionsServePipeline } from './pipeline.ts'; +import { openaiChatCompletionsServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; -import { createNonResponsesSourceStore } from '../responses/items/store.ts'; +import { createNonResponsesSourceStore } from '../openai-responses/items/store.ts'; import { move } from '@floway-dev/pipeline'; -import type { ChatCompletionsPayload } from '@floway-dev/protocols/chat-completions'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; /** A body the gateway could not read is reported in the words the protocol's own clients * parse, rather than as a fault of the gateway's. */ -const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: ChatCompletionsPayload } | { type: 'invalid'; message: string } => { +const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: OpenAIChatCompletionsPayload } | { type: 'invalid'; message: string } => { try { - return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as ChatCompletionsPayload }; + return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as OpenAIChatCompletionsPayload }; } catch (error) { return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } }; -export const chatCompletionsHttp = { +export const openaiChatCompletionsHttp = { generate: async (c: AuthedContext): Promise => { const ingress = await readIngress(c); const request = readRequest(ingress.body.bytes); @@ -54,21 +54,21 @@ export const chatCompletionsHttp = { return await serveThrough( c, prologue, - chatCompletionsServePipeline(payload), + openaiChatCompletionsServePipeline(payload), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'chatCompletions', - 'ingress.chat.chatCompletions.wantsStream': wantsStream, - 'ingress.chat.chatCompletions.wantsUsageChunk': payload.stream_options?.include_usage === true, - 'request.chat.chatCompletions': payload, + 'ingress.chat.openaiChatCompletions.wantsStream': wantsStream, + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': payload.stream_options?.include_usage === true, + 'request.chat.openaiChatCompletions': payload, 'serve.model': payload.model, }) as never, facts => { - const rendered = facts['response.chat.chatCompletions.rendered']; + const rendered = facts['response.chat.openaiChatCompletions.rendered']; if (isFrames(rendered)) return { frames: rendered }; return { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - facts => facts['response.chat.chatCompletions.streamedUsage'], + facts => facts['response.chat.openaiChatCompletions.streamedUsage'], ); }, }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index db2cf725b6..364bd063df 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -10,7 +10,7 @@ // The wire a candidate is dialled on is chosen per candidate rather than at assembly, and // because failover re-runs the whole suffix the next candidate re-picks its own. Its own // wire is a bare ending; the two translated ones are a handoff and then that protocol's own -// wire, and all three hand up `response.chat.chatCompletions` — so the stage above cannot +// wire, and all three hand up `response.chat.openaiChatCompletions` — so the stage above cannot // tell which ran. // // What sits *in* a wire rather than above the fork is the other half of the arrangement. A @@ -47,8 +47,8 @@ import { vendorKimiNormalizeForChatCompletions, vendorQwenNormalizeForChatCompletions, } from '../interceptors.ts'; -import { messagesWire } from '../messages/pipeline.ts'; -import { responsesWire } from '../responses/pipeline.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; +import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; import { createExternalImageLoader } from '../shared/external-image-loader.ts'; @@ -58,31 +58,31 @@ import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type Cha import { compose, defineStage, move, type Pipeline, type Stage, type Use } from '@floway-dev/pipeline'; import { CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, - chatCompletionsErrorPayloadMessage, - chatCompletionsProtocolFrameToSSEFrame, + openaiChatCompletionsErrorPayloadMessage, + openaiChatCompletionsProtocolFrameToSSEFrame, collectChatCompletionsProtocolEventsToResult, - type ChatCompletionsPayload, - type ChatCompletionsStreamEvent, -} from '@floway-dev/protocols/chat-completions'; + type OpenAIChatCompletionsPayload, + type OpenAIChatCompletionsStreamEvent, +} from '@floway-dev/protocols/openai-chat-completions'; import type { BillableUsage, ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateChatCompletionsViaMessages, translateChatCompletionsViaResponses } from '@floway-dev/translate'; /** `/v1/chat/completions` prefers its own wire, then the translated Messages path, then the * translated Responses path. */ -export const chatCompletionsTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); +export const openaiChatCompletionsTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); /** What this family adds to the chat space. */ -export interface ChatCompletionsFacts extends ChatFacts { +export interface OpenAIChatCompletionsFacts extends ChatFacts { /** What the client is actually sent — an object when it asked for one, SSE frames when it * asked to stream. The edge provides it, so a dump shows what the client received. */ - 'response.chat.chatCompletions.rendered': Record | AsyncIterable; + 'response.chat.openaiChatCompletions.rendered': Record | AsyncIterable; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ - 'response.chat.chatCompletions.streamedUsage': Promise | null; + 'response.chat.openaiChatCompletions.streamedUsage': Promise | null; } -type C = { [P in K]: ChatCompletionsFacts[P] }; +type C = { [P in K]: OpenAIChatCompletionsFacts[P] }; /** * The outermost edge. A chat answer is always a stream by the time it reaches here — the @@ -93,29 +93,29 @@ type C = { [P in K]: ChatCompletionsFacts[ * same frames that would have gone out are folded here instead. */ const emitChatCompletions = defineStage< - C<'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk'>, - C<'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk'>, - C<'ingress.chat.chatCompletions.wantsStream' | 'ingress.chat.chatCompletions.wantsUsageChunk' - | 'response.chat.chatCompletions' | 'response.http.headers'>, - C<'response.chat.chatCompletions.rendered' | 'response.http.status' | 'response.http.headers'>, + C<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, + C<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, + C<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk' + | 'response.chat.openaiChatCompletions' | 'response.http.headers'>, + C<'response.chat.openaiChatCompletions.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ name: 'emitChatCompletions', through: { request: { - needs: ['ingress.chat.chatCompletions.wantsStream', 'ingress.chat.chatCompletions.wantsUsageChunk'], + needs: ['ingress.chat.openaiChatCompletions.wantsStream', 'ingress.chat.openaiChatCompletions.wantsUsageChunk'], consumes: [], provides: [], }, response: { - needs: ['response.chat.chatCompletions', 'response.http.headers'], - consumes: ['response.chat.chatCompletions', 'response.http.headers'], - provides: ['response.chat.chatCompletions.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.chat.openaiChatCompletions', 'response.http.headers'], + consumes: ['response.chat.openaiChatCompletions', 'response.http.headers'], + provides: ['response.chat.openaiChatCompletions.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next, use) => { const back = await next(facts); - const { 'response.chat.chatCompletions': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.chat.openaiChatCompletions': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -126,7 +126,7 @@ const emitChatCompletions = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.chatCompletions.rendered': move(renderFailure( + 'response.chat.openaiChatCompletions.rendered': move(renderFailure( answer, () => ({ error: { message: answer.message, type: 'api_error' } }), )), @@ -137,7 +137,7 @@ const emitChatCompletions = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.chatCompletions.rendered': move(answer.body as Record), + 'response.chat.openaiChatCompletions.rendered': move(answer.body as Record), 'response.http.status': 200, }; } @@ -148,16 +148,16 @@ const emitChatCompletions = defineStage< // the frames — below the fold, and there would be nothing left to rewrite. const frames = recordFrames( wrapChatCompletionsAffinityEgress( - answer.frames as AsyncIterable>, + answer.frames as AsyncIterable>, affinityEgressOptions(use.gateway), ), use.gateway.dump, ); - if (!back['ingress.chat.chatCompletions.wantsStream']) { + if (!back['ingress.chat.openaiChatCompletions.wantsStream']) { return { ...rest, 'response.http.headers': forClient, - 'response.chat.chatCompletions.rendered': move( + 'response.chat.openaiChatCompletions.rendered': move( await collectChatCompletionsProtocolEventsToResult(frames) as unknown as Record, ), 'response.http.status': 200, @@ -166,7 +166,7 @@ const emitChatCompletions = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.chatCompletions.rendered': move(renderSSE(frames, back['ingress.chat.chatCompletions.wantsUsageChunk'])), + 'response.chat.openaiChatCompletions.rendered': move(renderSSE(frames, back['ingress.chat.openaiChatCompletions.wantsUsageChunk'])), 'response.http.status': 200, }; }, @@ -176,12 +176,12 @@ const emitChatCompletions = defineStage< * is shown it is the client's own question. The protocol owns which frame that is, and says * so by writing no frame at all. */ const renderSSE = ( - frames: AsyncIterable>, + frames: AsyncIterable>, includeUsageChunk: boolean, ): AsyncIterable => ({ [Symbol.asyncIterator]: () => (async function* () { for await (const frame of frames) { - const written = chatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk }); + const written = openaiChatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk }); if (written !== null) yield written; } })(), @@ -190,7 +190,7 @@ const renderSSE = ( /** * The wire. It dials Chat Completions and provides the answer at whichever family's response * key the chain above it reads — which is what makes it interchangeable with a translated - * chain: both hand up `response.chat.chatCompletions`, and the stage above cannot tell which + * chain: both hand up `response.chat.openaiChatCompletions`, and the stage above cannot tell which * ran. * * `streamedUsage` is the one key it is told. A wire hands up one reading whatever protocol it @@ -198,14 +198,14 @@ const renderSSE = ( * protocol's — so it is built with it rather than naming one of its own. */ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< - C<'request.chat.chatCompletions' | 'route.attempt' | 'ingress.http.headers'>, - C<'response.chat.chatCompletions' | 'response.usage.billable' | 'response.http.headers'> & Record, + C<'request.chat.openaiChatCompletions' | 'route.attempt' | 'ingress.http.headers'>, + C<'response.chat.openaiChatCompletions' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ name: 'callChatCompletionsUpstream', return: { provides: [ - 'response.chat.chatCompletions', + 'response.chat.openaiChatCompletions', streamedUsage, 'response.usage.billable', 'response.http.headers', @@ -220,7 +220,7 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< // What the record holds by now: the payload affinity materialized for this candidate, // as every stage between the fork and here has rewritten it — or, on a translated wire, // what the handoff put here. - const body = bodyForAttempt(facts['request.chat.chatCompletions'], candidate, applyRulesToUpstreamChatCompletions); + const body = bodyForAttempt(facts['request.chat.openaiChatCompletions'], candidate, applyRulesToUpstreamChatCompletions); let result; try { @@ -239,7 +239,7 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.chat.chatCompletions': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.openaiChatCompletions': { status: 502, message: error instanceof Error ? error.message : String(error) }, [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], @@ -258,7 +258,7 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } return move({ ...facts, - 'response.chat.chatCompletions': { + 'response.chat.openaiChatCompletions': { status: result.response.status, message: text, ...(parsed === undefined ? {} : { body: parsed }), @@ -275,7 +275,7 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< const metered = meterChatCompletions(result.events, identity, use.gateway.attempt); return move({ ...facts, - 'response.chat.chatCompletions': { kind: 'stream' as const, frames: metered.frames }, + 'response.chat.openaiChatCompletions': { kind: 'stream' as const, frames: metered.frames }, [streamedUsage]: metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], @@ -306,7 +306,7 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< * dialect on the request path: the canonical sentinel is emitted before a vendor spells it, * and the field an upstream would reject is gone before a vendor rewrites what is left. */ -export const chatCompletionsWire = (streamedUsage: string): readonly Stage[] => [ +export const openaiChatCompletionsWire = (streamedUsage: string): readonly Stage[] => [ includeUsageStreamOptionsForChatCompletions, normalizeUsageForChatCompletions, disableReasoningOnForcedToolChoiceForChatCompletions, @@ -320,35 +320,35 @@ export const chatCompletionsWire = (streamedUsage: string): readonly Stage[] => ]; /** This family's own reading, which every wire under it hands up. */ -const STREAMED_USAGE = 'response.chat.chatCompletions.streamedUsage'; +const STREAMED_USAGE = 'response.chat.openaiChatCompletions.streamedUsage'; /** The three wires `/v1/chat/completions` can be served on. Its own is the bare wire; each * translated one is a handoff and then the target protocol's own wire. */ -const chatCompletionsWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { +const openaiChatCompletionsWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { switch (target) { case 'chat-completions': - return compose('chatCompletionsNative', chatCompletionsWire(STREAMED_USAGE)); + return compose('openaiChatCompletionsNative', openaiChatCompletionsWire(STREAMED_USAGE)); case 'messages': - return compose('chatCompletionsViaMessages', [ + return compose('openaiChatCompletionsViaAnthropicMessages', [ handOff({ - from: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, - to: { request: 'request.chat.messages', response: 'response.chat.messages' }, + from: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, trip: async payload => await translateChatCompletionsViaMessages(payload, { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), }), }), - ...messagesWire(STREAMED_USAGE), + ...anthropicMessagesWire(STREAMED_USAGE), ]); case 'responses': - return compose('chatCompletionsViaResponses', [ + return compose('openaiChatCompletionsViaOpenAIResponses', [ handOff({ - from: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, - to: { request: 'request.chat.responses', response: 'response.chat.responses' }, + from: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, trip: async payload => await translateChatCompletionsViaResponses(payload, { model: candidate.model.id }), }), - ...responsesWire(STREAMED_USAGE), + ...openaiResponsesWire(STREAMED_USAGE), ]); } }; @@ -357,10 +357,10 @@ const chatCompletionsWireFor = (target: ChatTargetApi, candidate: ModelCandidate * pass and the client's stream is what drives it. Only a report carrying real counts * replaces the running figure, so a trailing empty usage frame cannot wipe a good one. */ const meterChatCompletions = ( - source: AsyncIterable>, + source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, -): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { +): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { let settle!: (outcome: StreamOutcome) => void; const outcome = new Promise(resolve => { settle = resolve; }); // Running out without the terminal frame is what "it did not finish" means, and it is known @@ -399,8 +399,8 @@ const meterChatCompletions = ( /** The frame that says this turn is over: the transport's own terminator, or an error the * upstream wrote into the stream instead of finishing it. */ -const isTerminal = (frame: ProtocolFrame): boolean => - frame.type === 'done' || (frame.type === 'event' && chatCompletionsErrorPayloadMessage(frame.event) !== null); +const isTerminal = (frame: ProtocolFrame): boolean => + frame.type === 'done' || (frame.type === 'event' && openaiChatCompletionsErrorPayloadMessage(frame.event) !== null); /** What one attempt is billable for. An upstream that reported nothing leaves no quantities * at all, which is a different statement from reporting zero — and a rate can depend on the @@ -416,13 +416,13 @@ const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModel /** A candidate that cannot serve *this* request is not a candidate — and what the client's * own turn carries decides the order the rest are tried in, which is why the narrowing is * built from the request rather than being a constant. */ -const narrowing = (payload: ChatCompletionsPayload): ChatNarrowing> => ({ - canServe: candidate => chatCompletionsTarget.canServe(candidate.model.endpoints), +const narrowing = (payload: OpenAIChatCompletionsPayload): ChatNarrowing> => ({ + canServe: candidate => openaiChatCompletionsTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeChatCompletionsAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /chat/completions endpoint.`, refuse: (status, message, reason) => ({ - 'response.chat.chatCompletions.streamedUsage': null, - 'response.chat.chatCompletions': { + 'response.chat.openaiChatCompletions.streamedUsage': null, + 'response.chat.openaiChatCompletions': { status, message, // What an OpenAI client reads: the condition's own type, and for a turn whose carried @@ -436,38 +436,38 @@ const narrowing = (payload: ChatCompletionsPayload): ChatNarrowing; -export type ChatCompletionsServeExit = C< - 'response.chat.chatCompletions.rendered' | 'response.chat.chatCompletions.streamedUsage' +export type OpenAIChatCompletionsServeExit = C< + 'response.chat.openaiChatCompletions.rendered' | 'response.chat.openaiChatCompletions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' >; -export const chatCompletionsServePipeline = (payload: ChatCompletionsPayload): Pipeline => - compose('chatCompletionsServe', [ +export const openaiChatCompletionsServePipeline = (payload: OpenAIChatCompletionsPayload): Pipeline => + compose('openaiChatCompletionsServe', [ emitChatCompletions, writeSettlement( - handedUp => isFailure((handedUp as { 'response.chat.chatCompletions'?: unknown })['response.chat.chatCompletions']), - handedUp => (handedUp as { 'response.chat.chatCompletions.streamedUsage'?: unknown })['response.chat.chatCompletions.streamedUsage'] !== null, + handedUp => isFailure((handedUp as { 'response.chat.openaiChatCompletions'?: unknown })['response.chat.openaiChatCompletions']), + handedUp => (handedUp as { 'response.chat.openaiChatCompletions.streamedUsage'?: unknown })['response.chat.openaiChatCompletions.streamedUsage'] !== null, ), resolveChatCandidates(narrowing(payload)), failover({ - failed: handedUp => isFailure((handedUp as { 'response.chat.chatCompletions'?: unknown })['response.chat.chatCompletions']), + failed: handedUp => isFailure((handedUp as { 'response.chat.openaiChatCompletions'?: unknown })['response.chat.openaiChatCompletions']), owns: [], }), - materializeAttempt('request.chat.chatCompletions'), + materializeAttempt('request.chat.openaiChatCompletions'), dialChatWire({ - source: 'request.chat.chatCompletions', - needs: ['request.chat.chatCompletions', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.chatCompletions', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], - pick: endpoints => chatCompletionsTarget.pick(endpoints), - wire: chatCompletionsWireFor, + source: 'request.chat.openaiChatCompletions', + needs: ['request.chat.openaiChatCompletions', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.openaiChatCompletions', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => openaiChatCompletionsTarget.pick(endpoints), + wire: openaiChatCompletionsWireFor, }), ]); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts b/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts index 8a1b04a7eb..49add24b87 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts @@ -6,11 +6,11 @@ import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { CanonicalResponsesPayload, ClientResponsesStreamEvent, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { CanonicalResponsesPayload, ClientResponsesStreamEvent, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; // Unix seconds, from the gateway's own request-start instant, so both resources // date a turn the same way. -export const responsesCreatedAt = (ctx: GatewayCtx): number => Math.floor(ctx.requestStartedAt / 1000); +export const openaiResponsesCreatedAt = (ctx: GatewayCtx): number => Math.floor(ctx.requestStartedAt / 1000); // The item lifecycle first resolves any partial terminal restatement. Affinity // then wraps routing metadata, and the client-output boundary stores each @@ -18,9 +18,9 @@ export const responsesCreatedAt = (ctx: GatewayCtx): number => Math.floor(ctx.re // ID to the downstream stream and snapshot. Every native Responses turn goes // through this half, whichever resource it answers with. export const wrapResponsesStatefulOutput = ( - frames: AsyncIterable>, + frames: AsyncIterable>, ctx: ChatGatewayCtx, -): AsyncIterable> => { +): AsyncIterable> => { const withObservedOutput = wrapResponsesObservedOutput(frames); const withAffinity = wrapResponsesAffinityEgress(withObservedOutput, affinityEgressOptions(ctx)); return wrapResponsesClientOutput(withAffinity, { @@ -33,12 +33,12 @@ export const wrapResponsesStatefulOutput = ( // completion. `/responses/compact` answers with `CompactResource`, so it stops // at the stateful half and completes that resource itself. export const wrapResponsesClientEgress = ( - frames: AsyncIterable>, + frames: AsyncIterable>, ctx: ChatGatewayCtx, request: CanonicalResponsesPayload, ): AsyncIterable> => wrapResponseResourceCompletion(wrapResponsesStatefulOutput(frames, ctx), { request, - createdAt: responsesCreatedAt(ctx), + createdAt: openaiResponsesCreatedAt(ctx), stored: ctx.store.writesState, }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts index ee87ba3f01..3c51b16fba 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts @@ -34,7 +34,7 @@ import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../shared/base64url-json.ts'; import { isJsonObject } from '../../../shared/json-helpers.ts'; -import type { CanonicalResponsesPayload, ResponsesInputItem, ResponsesOutputItem, ResponsesResult } from '@floway-dev/protocols/responses'; +import type { CanonicalResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; // The two vendored constants below (SUMMARIZATION_PROMPT and SUMMARY_PREFIX) // are the compactor system prompt and the handoff prefix openai/codex ships @@ -162,7 +162,7 @@ export const expandShimCompactionItems = (payload: CanonicalOpenAIResponsesPaylo // assistant message it just closed. A turn that closed nothing falls back to // the terminal, as the client-facing egress does. // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L237 -export const summaryTextFrom = (closed: Map, stated: readonly ResponsesOutputItem[]): string => { +export const summaryTextFrom = (closed: Map, stated: readonly OpenAIResponsesOutputItem[]): string => { const items = closed.size === 0 ? stated : [...closed].sort(([left], [right]) => left - right).map(([, item]) => item); @@ -176,7 +176,7 @@ export const summaryTextFrom = (closed: Map, stated return parts.join(''); }; -export const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: ResponsesResult): ResponsesResult => { +export const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: OpenAIResponsesResult): OpenAIResponsesResult => { // The prefix lives inside the blob so it round-trips atomically with the // summary — a downstream LLM sees `${SUMMARY_PREFIX}\n${summaryText}` in // one message and reads it as "another LLM's handoff", not as the human @@ -293,5 +293,5 @@ export const summarizationTurnFor = (payload: CanonicalResponsesPayload): Canoni // silently discards the conversation. export const EMPTY_SUMMARY_MESSAGE = 'Responses compact shim: the summarization turn closed no assistant text to summarize'; -export const containsCompactionTrigger = (input: readonly ResponsesInputItem[]): boolean => +export const containsCompactionTrigger = (input: readonly OpenAIResponsesInputItem[]): boolean => input.some(item => item.type === 'compaction_trigger'); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index 8242f100e5..c7eef24cec 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -11,13 +11,13 @@ // dialResponsesCompaction the ending: dials a compaction, or simulates one // // A second **operation** over this protocol rather than another wire under -// `responsesServePipeline`, which is why it is a chain of its own. Everything above the +// `openaiResponsesServePipeline`, which is why it is a chain of its own. Everything above the // ending it shares with generation, stage for stage, because a compaction is routed, pinned // and rewritten exactly as a turn is: the same membrane hydrates it, the same narrowing // orders its candidates, the same rules shape the body that goes out — the shim's own // expansion among them, because a compaction it wrote is echoed back into either entry. // -// The ending is where the two part. Two wires, both handing up `response.chat.responses`: +// The ending is where the two part. Two wires, both handing up `response.chat.openaiResponses`: // // - the compaction wire. An upstream whose own `/responses` endpoint compacts is asked to, // and answers with one envelope; the synthetic frames it becomes are what the stateful @@ -39,7 +39,7 @@ // a compaction whose caller declared a server tool is summarized without the ReAct loop wrapped // around it — the same gap the generate chain states, and for the same reason. -import { responsesCreatedAt, wrapResponsesStatefulOutput } from './client-output.ts'; +import { openaiResponsesCreatedAt, wrapResponsesStatefulOutput } from './client-output.ts'; import { completeResponsesCompaction } from './compaction-resource.ts'; import { syntheticEventsFromCompaction } from './items/output.ts'; import { @@ -49,13 +49,13 @@ import { hydrateStoredItems, internalErrorEnvelope, RESPONSES_STREAMED_USAGE, - responsesNarrowing, - responsesTarget, - responsesWireFor, - responsesWireRules, + openaiResponsesNarrowing, + openaiResponsesTarget, + openaiResponsesWireFor, + openaiResponsesWireRules, simulatesCompaction, summarizeForCompaction, - type ResponsesFacts, + type OpenAIResponsesFacts, } from './pipeline.ts'; import { billableUsageFromResponsesResult } from './usage.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; @@ -76,16 +76,16 @@ import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { collectResponsesProtocolEventsToResult, type CanonicalResponsesPayload, - type ResponsesStreamEvent, -} from '@floway-dev/protocols/responses'; + type OpenAIResponsesStreamEvent, +} from '@floway-dev/protocols/openai-responses'; import { providerModelOf } from '@floway-dev/provider'; -type R = { [P in K]: ResponsesFacts[P] }; +type R = { [P in K]: OpenAIResponsesFacts[P] }; /** What a compaction wire hands up. A compaction is a stream by the time it leaves either * ending — the envelope is expanded into the events the stateful half reads — so the value * arm this protocol's response key also admits is not one this chain ever produces. */ -type Compacted = { +type Compacted = { [P in K]: { readonly kind: 'stream'; readonly frames: AsyncIterable } | Failure; }; @@ -102,27 +102,27 @@ type Compacted = { const emitResponsesCompaction = defineStage< Record, Record, - Compacted<'response.chat.responses'> & R<'response.http.headers' | 'response.chat.responses.streamedUsage'>, - R<'response.chat.responses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.responses.streamedUsage'>, + Compacted<'response.chat.openaiResponses'> & R<'response.http.headers' | 'response.chat.openaiResponses.streamedUsage'>, + R<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.openaiResponses.streamedUsage'>, ChatServices >({ name: 'emitResponsesCompaction', through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.chat.responses', 'response.http.headers', 'response.chat.responses.streamedUsage'], - consumes: ['response.chat.responses', 'response.http.headers'], + needs: ['response.chat.openaiResponses', 'response.http.headers', 'response.chat.openaiResponses.streamedUsage'], + consumes: ['response.chat.openaiResponses', 'response.http.headers'], provides: [ - 'response.chat.responses.rendered', + 'response.chat.openaiResponses.rendered', 'response.http.status', 'response.http.headers', - 'response.chat.responses.streamedUsage', + 'response.chat.openaiResponses.streamedUsage', ], }, }, execute: async (facts, next, use) => { const back = await next(facts); - const { 'response.chat.responses': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.chat.openaiResponses': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -133,7 +133,7 @@ const emitResponsesCompaction = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(renderFailure( + 'response.chat.openaiResponses.rendered': move(renderFailure( answer, () => ({ error: { message: answer.message, type: 'api_error' } }), )), @@ -148,19 +148,19 @@ const emitResponsesCompaction = defineStage< // on the next line is the read. const persisted = await collectResponsesProtocolEventsToResult( recordFrames( - wrapResponsesStatefulOutput(answer.frames as AsyncIterable>, use.gateway), + wrapResponsesStatefulOutput(answer.frames as AsyncIterable>, use.gateway), use.gateway.dump, ), ); return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move( - completeResponsesCompaction(persisted, responsesCreatedAt(use.gateway)) as unknown as Record, + 'response.chat.openaiResponses.rendered': move( + completeResponsesCompaction(persisted, openaiResponsesCreatedAt(use.gateway)) as unknown as Record, ), 'response.http.status': 200, - 'response.chat.responses.streamedUsage': move( - withVerdict(rest['response.chat.responses.streamedUsage'], compactionFailed(persisted)), + 'response.chat.openaiResponses.streamedUsage': move( + withVerdict(rest['response.chat.openaiResponses.streamedUsage'], compactionFailed(persisted)), ), }; } catch (error) { @@ -170,9 +170,9 @@ const emitResponsesCompaction = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(internalErrorEnvelope(error)), + 'response.chat.openaiResponses.rendered': move(internalErrorEnvelope(error)), 'response.http.status': 502, - 'response.chat.responses.streamedUsage': move(withVerdict(rest['response.chat.responses.streamedUsage'], true)), + 'response.chat.openaiResponses.streamedUsage': move(withVerdict(rest['response.chat.openaiResponses.streamedUsage'], true)), }; } }, @@ -198,14 +198,14 @@ const withVerdict = (outcome: Promise | null, failed: boolean): P * which is what lets the stateful half above read either without knowing which ran. */ const callResponsesCompactUpstream = defineStage< - R<'request.chat.responses' | 'route.attempt' | 'ingress.http.headers'>, - Compacted<'response.chat.responses'> & R<'response.chat.responses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, + R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + Compacted<'response.chat.openaiResponses'> & R<'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'callResponsesCompactUpstream', return: { provides: [ - 'response.chat.responses', + 'response.chat.openaiResponses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers', @@ -217,7 +217,7 @@ const callResponsesCompactUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; + const asked = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; // Neither field belongs on this endpoint: `store` is a gateway-only snapshot hint the // compaction endpoint rejects, and `stream` describes how an answer would be delivered // where there is one body and no stream. @@ -241,7 +241,7 @@ const callResponsesCompactUpstream = defineStage< // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.chat.responses': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.openaiResponses': { status: 502, message: error instanceof Error ? error.message : String(error) }, [RESPONSES_STREAMED_USAGE]: null, 'response.usage.billable': [], 'response.http.headers': [], @@ -256,7 +256,7 @@ const callResponsesCompactUpstream = defineStage< try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } return move({ ...facts, - 'response.chat.responses': { + 'response.chat.openaiResponses': { status: result.response.status, message: text, ...(parsed === undefined ? {} : { body: parsed }), @@ -279,7 +279,7 @@ const callResponsesCompactUpstream = defineStage< const billable = [billedResponsesEntity(identity, billableUsageFromResponsesResult(result.result) ?? undefined)]; return move({ ...facts, - 'response.chat.responses': { kind: 'stream' as const, frames: syntheticEventsFromCompaction(result.result) }, + 'response.chat.openaiResponses': { kind: 'stream' as const, frames: syntheticEventsFromCompaction(result.result) }, // A compaction states its own counts, so what it billed is known before the frames are // read. It travels as a reading still to come rather than as one already settled, // because the verdict that goes with it is the edge's to add. @@ -293,17 +293,17 @@ const callResponsesCompactUpstream = defineStage< /** The generate fork, as the simulation reaches it. A summarization is an ordinary turn, so * it is dialled on the wires an ordinary turn is dialled on. */ const dialSummarizationWire = dialChatWire({ - source: 'request.chat.responses', - needs: ['request.chat.responses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.responses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], - pick: endpoints => responsesTarget.pick(endpoints), - wire: responsesWireFor, + source: 'request.chat.openaiResponses', + needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.openaiResponses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => openaiResponsesTarget.pick(endpoints), + wire: openaiResponsesWireFor, }); -const compactionWire: ChatWire = compose('responsesCompactNative', [...responsesWireRules, callResponsesCompactUpstream]); +const compactionWire: ChatWire = compose('openaiResponsesCompactNative', [...openaiResponsesWireRules, callResponsesCompactUpstream]); // The ending below picks this wire only for a candidate whose compactions are the shim's to // simulate, so the ask is already answered: every turn that reaches it is one to summarize. -const simulationWire: ChatWire = compose('responsesCompactSimulated', [summarizeForCompaction(() => true), dialSummarizationWire]); +const simulationWire: ChatWire = compose('openaiResponsesCompactSimulated', [summarizeForCompaction(() => true), dialSummarizationWire]); /** * Picks how this candidate's compaction is produced. @@ -327,7 +327,7 @@ const dialResponsesCompaction = defineStage< name: 'dialResponsesCompaction', into: { request: { - needs: ['route.attempt', 'request.chat.responses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + needs: ['route.attempt', 'request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], consumes: [], provides: [], }, @@ -338,7 +338,7 @@ const dialResponsesCompaction = defineStage< needs: [], consumes: [], provides: [ - 'response.chat.responses', + 'response.chat.openaiResponses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers', @@ -353,33 +353,33 @@ const dialResponsesCompaction = defineStage< }, }); -export type ResponsesCompactEntry = R< - 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.responses' | 'serve.model' +export type OpenAIResponsesCompactEntry = R< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.openaiResponses' | 'serve.model' >; -export type ResponsesCompactExit = R< - 'response.chat.responses.rendered' | 'response.chat.responses.streamedUsage' +export type OpenAIResponsesCompactExit = R< + 'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage' | 'response.http.status' | 'response.http.headers' >; -export const responsesCompactPipeline = (payload: CanonicalResponsesPayload): Pipeline => { +export const openaiResponsesCompactPipeline = (payload: CanonicalResponsesPayload): Pipeline => { // One cell per run, written by the stage directly above the one that reads it — the same // hand-across the generate chain makes, for the same reason: the resolver takes its // narrowing at assembly, before any fact exists. let prepared = payload; - return compose('responsesCompact', [ + return compose('openaiResponsesCompact', [ emitResponsesCompaction, writeSettlement( - handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), - handedUp => (handedUp as { 'response.chat.responses.streamedUsage'?: unknown })['response.chat.responses.streamedUsage'] !== null, + handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), + handedUp => (handedUp as { 'response.chat.openaiResponses.streamedUsage'?: unknown })['response.chat.openaiResponses.streamedUsage'] !== null, ), hydrateStoredItems(payload, hydrated => { prepared = hydrated; }), - resolveChatCandidates(responsesNarrowing(() => prepared)), + resolveChatCandidates(openaiResponsesNarrowing(() => prepared)), failover({ - failed: handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), + failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), owns: [], }), - materializeAttempt('request.chat.responses'), + materializeAttempt('request.chat.openaiResponses'), beginStoredAttempt, expandShimCompactions, dialResponsesCompaction, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/errors.ts b/packages/gateway/src/data-plane/chat/openai-responses/errors.ts index 374fbd0457..c2fe6ddbea 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/errors.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/errors.ts @@ -4,4 +4,4 @@ import type { ChatServeFailure } from '../shared/errors.ts'; * by id which the store cannot resolve. It is thrown from inside the membrane's hydration * rather than returned, because hydration walks the input and the item that is missing is * found several frames below where the answer is written. */ -export type ResponsesServeFailure = ChatServeFailure | { readonly kind: 'item-not-found'; readonly itemId: string }; +export type OpenAIResponsesServeFailure = ChatServeFailure | { readonly kind: 'item-not-found'; readonly itemId: string }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/http.ts b/packages/gateway/src/data-plane/chat/openai-responses/http.ts index 59bc933b39..ba1738c424 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/http.ts @@ -7,15 +7,15 @@ // the stored-items membrane included — which is why a continuation that does not resolve is // answered by the chain on both routes rather than caught as a throw on either. -import { responsesCompactPipeline } from './compact.ts'; +import { openaiResponsesCompactPipeline } from './compact.ts'; import { createResponsesHttpStore } from './items/store.ts'; -import { responsesServePipeline } from './pipeline.ts'; +import { openaiResponsesServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; import { move } from '@floway-dev/pipeline'; -import type { CanonicalResponsesPayload, ResponsesRequestPayload } from '@floway-dev/protocols/responses'; +import type { CanonicalResponsesPayload, OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; /** The read, as a value rather than a throw, because a pipelined entry decides what to do @@ -29,7 +29,7 @@ const readRequest = (bytes: Uint8Array): | { type: 'ok'; payload: CanonicalResponsesPayload } | { type: 'invalid'; message: string; param?: string; code?: string } => { try { - return { type: 'ok', payload: canonicalizeResponsesPayload(JSON.parse(new TextDecoder().decode(bytes)) as ResponsesRequestPayload) }; + return { type: 'ok', payload: canonicalizeResponsesPayload(JSON.parse(new TextDecoder().decode(bytes)) as OpenAIResponsesRequestPayload) }; } catch (error) { if (!(error instanceof TranslatorInputError)) { return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; @@ -67,7 +67,7 @@ const refuse = (c: AuthedContext, ingress: Ingress, invalid: { message: string; return finalizeGatewayResponse(refused.gateway, invalidRequestResponse(invalid)); }; -export const responsesHttp = { +export const openaiResponsesHttp = { generate: async (c: AuthedContext): Promise => { const ingress = await readIngress(c); const request = readRequest(ingress.body.bytes); @@ -84,20 +84,20 @@ export const responsesHttp = { return await serveThrough( c, prologue, - responsesServePipeline(payload), + openaiResponsesServePipeline(payload), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'responses', - 'ingress.chat.responses.wantsStream': wantsStream, - 'request.chat.responses': payload, + 'ingress.chat.openaiResponses.wantsStream': wantsStream, + 'request.chat.openaiResponses': payload, 'serve.model': payload.model, }) as never, facts => { - const rendered = facts['response.chat.responses.rendered']; + const rendered = facts['response.chat.openaiResponses.rendered']; if (isFrames(rendered)) return { frames: rendered }; return { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - facts => facts['response.chat.responses.streamedUsage'], + facts => facts['response.chat.openaiResponses.streamedUsage'], ); }, @@ -116,17 +116,17 @@ export const responsesHttp = { return await serveThrough( c, prologue, - responsesCompactPipeline(payload), + openaiResponsesCompactPipeline(payload), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'responses', - 'request.chat.responses': payload, + 'request.chat.openaiResponses': payload, 'serve.model': payload.model, }) as never, // A compaction is one resource however the turn went, so there is never a stream to // write: the frames it came as were read into that resource before the run answered. - facts => ({ body: JSON.stringify(facts['response.chat.responses.rendered']), contentType: 'application/json' }), - facts => facts['response.chat.responses.streamedUsage'], + facts => ({ body: JSON.stringify(facts['response.chat.openaiResponses.rendered']), contentType: 'application/json' }), + facts => facts['response.chat.openaiResponses.streamedUsage'], ); }, }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts index 88eb70c54b..300c417546 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts @@ -1,13 +1,13 @@ import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; import type { Interceptor } from '@floway-dev/interceptor'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import type { ExecuteResult, ResponsesInvocation } from '@floway-dev/provider'; +import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { ExecuteResult, OpenAIResponsesInvocation } from '@floway-dev/provider'; -export type { ResponsesInvocation }; +export type { OpenAIResponsesInvocation }; -export type ResponsesInterceptor = Interceptor< - ResponsesInvocation, +export type OpenAIResponsesInterceptor = Interceptor< + OpenAIResponsesInvocation, ChatGatewayCtx, - ExecuteResult> + ExecuteResult> >; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts index 690b3701a4..e8e52776c3 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts @@ -1,9 +1,9 @@ -import { createResponsesStorageKey, hashResponsesItem, responsesItemId } from './identity.ts'; +import { createResponsesStorageKey, hashResponsesItem, openaiResponsesItemId } from './identity.ts'; import { getRepo } from '../../../../repo/index.ts'; -import { assertSameStoredResponsesItem, cloneStoredResponsesItem, cloneStoredResponsesSnapshot, compareResponsesItemsByFreshness, scopedResponsesKey } from '../../../../repo/responses-clone.ts'; -import { quantizeResponsesRefreshedAt, responsesStateCutoff } from '../../../../repo/responses-retention.ts'; +import { assertSameStoredResponsesItem, cloneStoredResponsesItem, cloneStoredResponsesSnapshot, compareResponsesItemsByFreshness, scopedResponsesKey } from '../../../../repo/openai-responses-clone.ts'; +import { quantizeResponsesRefreshedAt, openaiResponsesStateCutoff } from '../../../../repo/openai-responses-retention.ts'; import type { ApiKey, Repo, StoredResponsesItem, StoredResponsesSnapshot } from '../../../../repo/types.ts'; -import type { ResponsesInputItem } from '@floway-dev/protocols/responses'; +import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; interface StatefulResponsesItemLookup { readonly apiKeyId: string; @@ -25,17 +25,17 @@ interface LayeredStatefulResponsesStoreOptions { readonly writes: readonly StatefulResponsesBacking[]; } -type ResponsesSnapshotMode = 'append' | 'replace'; +type OpenAIResponsesSnapshotMode = 'append' | 'replace'; export interface StatefulResponsesStore { readonly apiKeyId: string; readonly writesState: boolean; loadSnapshot(id: string): Promise; - loadInputItems(sourceItems: readonly ResponsesInputItem[], inputItemsToStage: readonly ResponsesInputItem[]): Promise; + loadInputItems(sourceItems: readonly OpenAIResponsesInputItem[], inputItemsToStage: readonly OpenAIResponsesInputItem[]): Promise; getItemById(id: string): StoredResponsesItem | undefined; - stageInputItems(items: readonly ResponsesInputItem[]): Promise; + stageInputItems(items: readonly OpenAIResponsesInputItem[]): Promise; persistOutputItem(row: StoredResponsesItem): Promise; - commitSnapshot(responseId: string, mode: ResponsesSnapshotMode, outputItemIds: readonly string[]): Promise; + commitSnapshot(responseId: string, mode: OpenAIResponsesSnapshotMode, outputItemIds: readonly string[]): Promise; // Per-attempt transient state. `beginAttempt` reseeds the private-payload // scratchpad from hydrated items; interceptors can add server-only state // during the turn and output capture persists it with the exact wire item. @@ -51,7 +51,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { private previousSnapshotItemIds: string[] = []; private readonly committedItemIds = new Set(); private readonly privatePayloads = new Map(); - private readonly inputItemHashes = new WeakMap(); + private readonly inputItemHashes = new WeakMap(); constructor(private readonly options: LayeredStatefulResponsesStoreOptions) {} @@ -90,18 +90,18 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { } async loadInputItems( - sourceItems: readonly ResponsesInputItem[], - inputItemsToStage: readonly ResponsesInputItem[], + sourceItems: readonly OpenAIResponsesInputItem[], + inputItemsToStage: readonly OpenAIResponsesInputItem[], ): Promise { const ids = new Set(); for (const item of sourceItems) { - const id = responsesItemId(item); + const id = openaiResponsesItemId(item); if (id !== null) ids.add(id); } const itemHashes = new Set(); for (const item of this.writesState ? inputItemsToStage : []) { if (item.type === 'item_reference' || item.type === 'compaction_trigger') continue; - if (responsesItemId(item) !== null) continue; + if (openaiResponsesItemId(item) !== null) continue; itemHashes.add(await this.hashInputItem(item)); } await this.loadItems({ ids: [...ids], itemHashes: [...itemHashes] }); @@ -112,7 +112,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { return row === undefined ? undefined : cloneStoredResponsesItem(row); } - async stageInputItems(items: readonly ResponsesInputItem[]): Promise { + async stageInputItems(items: readonly OpenAIResponsesInputItem[]): Promise { if (!this.writesState) return; for (const item of items) await this.stageInputItem(item); } @@ -127,7 +127,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { this.rememberItem(cloned); } - async commitSnapshot(responseId: string, mode: ResponsesSnapshotMode, outputItemIds: readonly string[]): Promise { + async commitSnapshot(responseId: string, mode: OpenAIResponsesSnapshotMode, outputItemIds: readonly string[]): Promise { if (this.options.writes.length === 0) return; const itemIds = mode === 'replace' ? [...outputItemIds] @@ -180,7 +180,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { } } - private async stageInputItem(item: ResponsesInputItem): Promise { + private async stageInputItem(item: OpenAIResponsesInputItem): Promise { if (item.type === 'compaction_trigger') return; if (item.type === 'item_reference') { const row = this.loadedItems.get(item.id); @@ -189,7 +189,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { return; } - const id = responsesItemId(item); + const id = openaiResponsesItemId(item); if (id !== null) { const row = this.loadedItems.get(id); if (row !== undefined) { @@ -227,7 +227,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { this.rememberItem(row); } - private async hashInputItem(item: ResponsesInputItem): Promise { + private async hashInputItem(item: OpenAIResponsesInputItem): Promise { const cached = this.inputItemHashes.get(item); if (cached !== undefined) return cached; const hash = await hashResponsesItem(item); @@ -268,13 +268,13 @@ export class RepoStatefulResponsesBacking implements StatefulResponsesBacking { requestStartedAt: number, retentionSeconds: number, ) { - this.earliestVisibleCutoff = responsesStateCutoff(requestStartedAt, retentionSeconds); + this.earliestVisibleCutoff = openaiResponsesStateCutoff(requestStartedAt, retentionSeconds); } async lookupItems(query: StatefulResponsesItemLookup): Promise { const [byId, byItemHash] = await Promise.all([ - this.getRepo().responsesItems.lookupMany(query.apiKeyId, query.ids, this.earliestVisibleCutoff), - this.getRepo().responsesItems.lookupManyByItemHash(query.apiKeyId, query.itemHashes, this.earliestVisibleCutoff), + this.getRepo().openaiResponsesItems.lookupMany(query.apiKeyId, query.ids, this.earliestVisibleCutoff), + this.getRepo().openaiResponsesItems.lookupManyByItemHash(query.apiKeyId, query.itemHashes, this.earliestVisibleCutoff), ]); const rows = new Map(); for (const row of [...byId, ...byItemHash]) rows.set(scopedResponsesKey(row.apiKeyId, row.id), row); @@ -282,19 +282,19 @@ export class RepoStatefulResponsesBacking implements StatefulResponsesBacking { } async insertItems(items: readonly StoredResponsesItem[]): Promise { - await this.getRepo().responsesItems.insertMany(items, this.earliestVisibleCutoff); + await this.getRepo().openaiResponsesItems.insertMany(items, this.earliestVisibleCutoff); } async refreshItems(items: readonly StoredResponsesItem[], refreshedAt: number): Promise { - await this.getRepo().responsesItems.refreshMany(items, refreshedAt, this.earliestVisibleCutoff); + await this.getRepo().openaiResponsesItems.refreshMany(items, refreshedAt, this.earliestVisibleCutoff); } async lookupSnapshot(apiKeyId: string, id: string): Promise { - return await this.getRepo().responsesSnapshots.lookup(apiKeyId, id, this.earliestVisibleCutoff); + return await this.getRepo().openaiResponsesSnapshots.lookup(apiKeyId, id, this.earliestVisibleCutoff); } async insertSnapshot(snapshot: StoredResponsesSnapshot): Promise { - await this.getRepo().responsesSnapshots.insert(snapshot); + await this.getRepo().openaiResponsesSnapshots.insert(snapshot); } } @@ -371,14 +371,14 @@ export class MemoryStatefulResponsesBacking implements StatefulResponsesBacking } } -type ResponsesStatePolicy = Pick; +type OpenAIResponsesStatePolicy = Pick; -const createDurableBacking = (apiKey: ResponsesStatePolicy, requestStartedAt: number): RepoStatefulResponsesBacking | null => - apiKey.responsesRetentionSeconds === 0 +const createDurableBacking = (apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number): RepoStatefulResponsesBacking | null => + apiKey.openaiResponsesRetentionSeconds === 0 ? null - : new RepoStatefulResponsesBacking(getRepo, requestStartedAt, apiKey.responsesRetentionSeconds); + : new RepoStatefulResponsesBacking(getRepo, requestStartedAt, apiKey.openaiResponsesRetentionSeconds); -export const createResponsesHttpStore = (apiKey: ResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore => { +export const createResponsesHttpStore = (apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore => { const durable = createDurableBacking(apiKey, requestStartedAt); return new LayeredStatefulResponsesStore({ apiKeyId: apiKey.id, @@ -397,7 +397,7 @@ export const createNonResponsesSourceStore = (apiKeyId: string): StatefulRespons new LayeredStatefulResponsesStore({ apiKeyId, reads: [], writes: [] }); export const createResponsesWsSession = (): { - createStore(apiKey: ResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore; + createStore(apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore; evictSnapshot(apiKeyId: string, id: string): void; } => { // "Servers SHOULD keep the most recent previous-response state in @@ -408,7 +408,7 @@ export const createResponsesWsSession = (): { // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L125 const local = new MemoryStatefulResponsesBacking(); return { - createStore(apiKey: ResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore { + createStore(apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore { const durable = createDurableBacking(apiKey, requestStartedAt); // Session-local state is the first store:true collision gate. Writing it // first prevents a rejected local history from creating a durable row. diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 88724218b2..a19f500bf9 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -11,7 +11,7 @@ // summarizeForCompaction answers a turn that asked for a compaction with one // dialChatWire the ending: picks this candidate's wire and hands into it // -// Three wires, all handing up `response.chat.responses`: this protocol's own, and the two +// Three wires, all handing up `response.chat.openaiResponses`: this protocol's own, and the two // translated ones — Responses via Messages and Responses via Chat Completions — each a // handoff followed by that protocol's own wire. What sits *in* a wire rather than above the // fork is a rule that speaks about that wire: the role rewrite and the cache-bucket fold both @@ -54,7 +54,7 @@ import { summarizationTurnFor, summaryTextFrom, } from './compact-shim.ts'; -import type { ResponsesServeFailure } from './errors.ts'; +import type { OpenAIResponsesServeFailure } from './errors.ts'; import { hydrateResponsesPayload } from './items/hydrate.ts'; import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; import { syntheticEventsFromResult } from './items/output.ts'; @@ -71,7 +71,7 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; -import { chatCompletionsWire } from '../chat-completions/pipeline.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { @@ -82,7 +82,7 @@ import { vendorDeepSeekNormalizeForResponses, vendorQwenNormalizeForResponses, } from '../interceptors.ts'; -import { messagesWire } from '../messages/pipeline.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; import { tryCatchChatServeFailure } from '../shared/errors.ts'; import { createExternalImageLoader } from '../shared/external-image-loader.ts'; @@ -95,29 +95,29 @@ import { collectResponsesProtocolEventsToResult, createRandomResponsesItemId, isResponsesTerminalEvent, - responsesProtocolFrameToSSEFrame, + openaiResponsesProtocolFrameToSSEFrame, RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type ClientResponseResource, type ClientResponsesStreamEvent, - type ResponsesOutputItem, - type ResponsesStreamEvent, -} from '@floway-dev/protocols/responses'; + type OpenAIResponsesOutputItem, + type OpenAIResponsesStreamEvent, +} from '@floway-dev/protocols/openai-responses'; import { providerModelOf, toInternalDebugError, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateResponsesViaChatCompletions, translateResponsesViaMessages } from '@floway-dev/translate'; /** `/v1/responses` prefers its own wire, then the translated Messages path, then the * translated Chat Completions path. */ -export const responsesTarget = chatTargetPicker(['responses', 'messages', 'chat-completions']); +export const openaiResponsesTarget = chatTargetPicker(['responses', 'messages', 'chat-completions']); /** How the transport that opened the run frames a streamed answer. Both carry this * protocol's own events and neither adds or drops one: SSE is the format an HTTP body is * written in, terminator and all, and a WebSocket turn writes each event as a text frame of * its own — so the transport that owns the socket takes the events and frames them itself. */ -export type ResponsesStreamFraming = 'sse' | 'events'; +export type OpenAIResponsesStreamFraming = 'sse' | 'events'; /** What this family adds to the chat space. */ -export interface ResponsesFacts extends ChatFacts { +export interface OpenAIResponsesFacts extends ChatFacts { /** Server-only state the gateway once attached to an item it emitted, keyed by that * item's id, as the rows this turn hydrated carry it. It never reaches an upstream: it is * what lets an item this turn re-emits be stored with the state it already had. @@ -125,17 +125,17 @@ export interface ResponsesFacts extends ChatFacts { * Pairs rather than a `Map`, for the same reason the header keys are: a `Map` has no own * properties, so it would be written into the dump as an empty object and the record would * say the turn hydrated nothing. */ - 'request.chat.responses.privatePayloads': readonly (readonly [string, unknown])[]; + 'request.chat.openaiResponses.privatePayloads': readonly (readonly [string, unknown])[]; /** What the client is actually sent — an object when it asked for one, and the stream it * asked to be streamed, in the framing its transport writes. The edge provides it, so a * dump shows what the client received. */ - 'response.chat.responses.rendered': Record | AsyncIterable | AsyncIterable>; + 'response.chat.openaiResponses.rendered': Record | AsyncIterable | AsyncIterable>; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ - 'response.chat.responses.streamedUsage': Promise | null; + 'response.chat.openaiResponses.streamedUsage': Promise | null; } -type R = { [P in K]: ResponsesFacts[P] }; +type R = { [P in K]: OpenAIResponsesFacts[P] }; /** * The outermost edge. What reaches it is usually a stream — the upstream speaks SSE whatever @@ -155,25 +155,25 @@ type R = { [P in K]: ResponsesFacts[P] }; * Only the last step differs: `renderSSE` is a wire format written over an HTTP body, and a * transport that frames each event itself takes the events it would have been written from. */ -const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStreamFraming) => defineStage< - R<'ingress.chat.responses.wantsStream'>, - R<'ingress.chat.responses.wantsStream'>, - R<'ingress.chat.responses.wantsStream' | 'response.chat.responses' | 'response.http.headers'>, - R<'response.chat.responses.rendered' | 'response.http.status' | 'response.http.headers'>, +const emitResponses = (client: CanonicalResponsesPayload, framing: OpenAIResponsesStreamFraming) => defineStage< + R<'ingress.chat.openaiResponses.wantsStream'>, + R<'ingress.chat.openaiResponses.wantsStream'>, + R<'ingress.chat.openaiResponses.wantsStream' | 'response.chat.openaiResponses' | 'response.http.headers'>, + R<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ name: 'emitResponses', through: { - request: { needs: ['ingress.chat.responses.wantsStream'], consumes: [], provides: [] }, + request: { needs: ['ingress.chat.openaiResponses.wantsStream'], consumes: [], provides: [] }, response: { - needs: ['response.chat.responses', 'response.http.headers'], - consumes: ['response.chat.responses', 'response.http.headers'], - provides: ['response.chat.responses.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.chat.openaiResponses', 'response.http.headers'], + consumes: ['response.chat.openaiResponses', 'response.http.headers'], + provides: ['response.chat.openaiResponses.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next, use) => { const back = await next(facts); - const { 'response.chat.responses': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.chat.openaiResponses': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -184,7 +184,7 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(renderFailure( + 'response.chat.openaiResponses.rendered': move(renderFailure( answer, () => ({ error: { message: answer.message, type: 'api_error' } }), )), @@ -197,7 +197,7 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(answer.body as Record), + 'response.chat.openaiResponses.rendered': move(answer.body as Record), 'response.http.status': 200, }; } @@ -210,7 +210,7 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre // schema requires of it. It runs before the fold rather than beside it, so a client that // did not ask to stream is answered with the object the persisted frames add up to. const egress = wrapResponsesClientEgress( - answer.frames as AsyncIterable>, + answer.frames as AsyncIterable>, use.gateway, client, ); @@ -224,12 +224,12 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre // what it hands up. Reading is what records, so a transport that stopped early records // exactly what it took. const frames = recordFrames(egress, use.gateway.dump); - if (!back['ingress.chat.responses.wantsStream']) { + if (!back['ingress.chat.openaiResponses.wantsStream']) { try { return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move( + 'response.chat.openaiResponses.rendered': move( await collectResponsesProtocolEventsToResult(frames) as unknown as Record, ), 'response.http.status': 200, @@ -241,7 +241,7 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(internalErrorEnvelope(error)), + 'response.chat.openaiResponses.rendered': move(internalErrorEnvelope(error)), 'response.http.status': 502, }; } @@ -249,7 +249,7 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: ResponsesStre return { ...rest, 'response.http.headers': forClient, - 'response.chat.responses.rendered': move(framing === 'sse' ? renderSSE(frames) : frames), + 'response.chat.openaiResponses.rendered': move(framing === 'sse' ? renderSSE(frames) : frames), 'response.http.status': 200, }; }, @@ -325,15 +325,15 @@ const renderSSE = (frames: AsyncIterable defineStage< - R<'request.chat.responses' | 'route.attempt' | 'ingress.http.headers'>, - R<'response.chat.responses' | 'response.usage.billable' | 'response.http.headers'> & Record, + R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + R<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ name: 'callResponsesUpstream', return: { provides: [ - 'response.chat.responses', + 'response.chat.openaiResponses', streamedUsage, 'response.usage.billable', 'response.http.headers', @@ -372,7 +372,7 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< // // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to, which is the one a wire takes. - const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; + const asked = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; const body = bodyForAttempt(asked, candidate, applyRulesToUpstreamResponses); let result; @@ -393,7 +393,7 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.chat.responses': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.chat.openaiResponses': { status: 502, message: error instanceof Error ? error.message : String(error) }, [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], @@ -412,7 +412,7 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } return move({ ...facts, - 'response.chat.responses': { + 'response.chat.openaiResponses': { status: result.response.status, message: text, ...(parsed === undefined ? {} : { body: parsed }), @@ -434,7 +434,7 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< if (result.action === 'compact') { return move({ ...facts, - 'response.chat.responses': { kind: 'value' as const, body: result.result }, + 'response.chat.openaiResponses': { kind: 'value' as const, body: result.result }, [streamedUsage]: null, 'response.usage.billable': [billedResponsesEntity(identity, billableUsageFromResponsesResult(result.result) ?? undefined)], 'response.http.headers': [], @@ -444,7 +444,7 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< const metered = meterResponses(result.events, identity, use.gateway.attempt); return move({ ...facts, - 'response.chat.responses': { kind: 'stream' as const, frames: metered.frames }, + 'response.chat.openaiResponses': { kind: 'stream' as const, frames: metered.frames }, [streamedUsage]: metered.outcome, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], @@ -465,24 +465,24 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< * the disagreement in the first place. */ const normalizeAssistantContentForResponses = defineStage< - R<'request.chat.responses'>, - R<'request.chat.responses'>, + R<'request.chat.openaiResponses'>, + R<'request.chat.openaiResponses'>, Record, Record, ChatServices >({ name: 'normalizeAssistantContentForResponses', through: { - request: { needs: ['request.chat.responses'], consumes: [], provides: ['request.chat.responses'] }, + request: { needs: ['request.chat.openaiResponses'], consumes: [], provides: ['request.chat.openaiResponses'] }, response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next) => { - const payload = facts['request.chat.responses'] as CanonicalResponsesPayload; + const payload = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; const input = normalizeAssistantInputText(payload.input); // A rewrite that changed nothing hands the same payload on, so the record shows no // change where none happened. if (input === payload.input) return await next(facts); - return await next({ ...facts, 'request.chat.responses': move({ ...payload, input }) }); + return await next({ ...facts, 'request.chat.openaiResponses': move({ ...payload, input }) }); }, }); @@ -499,7 +499,7 @@ const normalizeAssistantContentForResponses = defineStage< * They are named apart from the dial because a compaction is dialled differently and is * subject to the same three. */ -export const responsesWireRules: readonly Stage[] = [ +export const openaiResponsesWireRules: readonly Stage[] = [ normalizeAssistantContentForResponses, disableReasoningOnForcedToolChoiceForResponses, applyRoleCompatibilityToResponses, @@ -510,41 +510,41 @@ export const responsesWireRules: readonly Stage[] = [ ]; /** The Responses wire, as the chain that dials it. */ -export const responsesWire = (streamedUsage: string): readonly Stage[] => [ - ...responsesWireRules, +export const openaiResponsesWire = (streamedUsage: string): readonly Stage[] => [ + ...openaiResponsesWireRules, callResponsesUpstream(streamedUsage), ]; /** This family's own reading, which every wire under it hands up. */ -export const RESPONSES_STREAMED_USAGE = 'response.chat.responses.streamedUsage'; +export const RESPONSES_STREAMED_USAGE = 'response.chat.openaiResponses.streamedUsage'; /** The three wires `/v1/responses` can be served on. Its own is the bare wire; each translated * one is a handoff and then the target protocol's own wire. */ -export const responsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { +export const openaiResponsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { switch (target) { case 'responses': - return compose('responsesNative', responsesWire(RESPONSES_STREAMED_USAGE)); + return compose('openaiResponsesNative', openaiResponsesWire(RESPONSES_STREAMED_USAGE)); case 'messages': - return compose('responsesViaMessages', [ + return compose('openaiResponsesViaAnthropicMessages', [ handOff({ - from: { request: 'request.chat.responses', response: 'response.chat.responses' }, - to: { request: 'request.chat.messages', response: 'response.chat.messages' }, + from: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, + to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, trip: async payload => await translateResponsesViaMessages(payload, { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), }), }), - ...messagesWire(RESPONSES_STREAMED_USAGE), + ...anthropicMessagesWire(RESPONSES_STREAMED_USAGE), ]); case 'chat-completions': - return compose('responsesViaChatCompletions', [ + return compose('openaiResponsesViaOpenAIChatCompletions', [ handOff({ - from: { request: 'request.chat.responses', response: 'response.chat.responses' }, - to: { request: 'request.chat.chatCompletions', response: 'response.chat.chatCompletions' }, + from: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, + to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, trip: async payload => await translateResponsesViaChatCompletions(payload, { model: candidate.model.id }), }), - ...chatCompletionsWire(RESPONSES_STREAMED_USAGE), + ...openaiChatCompletionsWire(RESPONSES_STREAMED_USAGE), ]); } }; @@ -554,10 +554,10 @@ export const responsesWireFor = (target: ChatTargetApi, candidate: ModelCandidat * lifecycle envelopes, and only one carrying real counts replaces the running figure, so an * envelope that states none cannot wipe a good reading. */ const meterResponses = ( - source: AsyncIterable>, + source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, -): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { +): { readonly frames: AsyncIterable>; readonly outcome: Promise } => { let settle!: (outcome: StreamOutcome) => void; const outcome = new Promise(resolve => { settle = resolve; }); // Running out without the terminal frame is what "it did not finish" means, and it is known @@ -626,13 +626,13 @@ export const billedResponsesEntity = (identity: TelemetryModelIdentity, usage: B * client never sent, and a turn is pinned by what its items carry. The narrowing is built * at assembly, before any fact exists, so the membrane hands the prepared payload across * through the run's own cell rather than through the record. */ -export const responsesNarrowing = (prepared: () => CanonicalResponsesPayload): ChatNarrowing> => ({ - canServe: candidate => responsesTarget.canServe(candidate.model.endpoints), +export const openaiResponsesNarrowing = (prepared: () => CanonicalResponsesPayload): ChatNarrowing> => ({ + canServe: candidate => openaiResponsesTarget.canServe(candidate.model.endpoints), affinity: async gateway => await analyzeResponsesAffinity(prepared(), gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /responses endpoint.`, refuse: (status, message, reason) => ({ - 'response.chat.responses.streamedUsage': null, - 'response.chat.responses': { + 'response.chat.openaiResponses.streamedUsage': null, + 'response.chat.openaiResponses': { status, message, // What an OpenAI client reads: the condition's own type, and for a turn whose carried @@ -646,7 +646,7 @@ export const responsesNarrowing = (prepared: () => CanonicalResponsesPayload): C }, }, }), - refuses: ['response.chat.responses', 'response.chat.responses.streamedUsage'], + refuses: ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage'], }); /** A refusal the membrane makes on its own, in the shape every other refusal in this chain @@ -656,11 +656,11 @@ const refuseStoredItems = ( status: number, message: string, extra: { readonly param: string; readonly code: string | null }, -): R<'response.chat.responses' | 'response.chat.responses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'> => ({ +): R<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'> => ({ 'response.usage.billable': [], 'response.http.headers': [], - 'response.chat.responses.streamedUsage': null, - 'response.chat.responses': { + 'response.chat.openaiResponses.streamedUsage': null, + 'response.chat.openaiResponses': { status, message, envelope: { error: { message, type: 'invalid_request_error', ...extra } }, @@ -687,26 +687,26 @@ export const hydrateStoredItems = ( client: CanonicalResponsesPayload, prepared: (payload: CanonicalResponsesPayload) => void, ) => defineStage< - R<'request.chat.responses'>, - R<'request.chat.responses' | 'request.chat.responses.privatePayloads'>, + R<'request.chat.openaiResponses'>, + R<'request.chat.openaiResponses' | 'request.chat.openaiResponses.privatePayloads'>, Record, Record, - R<'response.chat.responses' | 'response.chat.responses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, + R<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'hydrateStoredItems', through: { request: { - needs: ['request.chat.responses'], + needs: ['request.chat.openaiResponses'], consumes: [], - provides: ['request.chat.responses', 'request.chat.responses.privatePayloads'], + provides: ['request.chat.openaiResponses', 'request.chat.openaiResponses.privatePayloads'], }, response: { needs: [], consumes: [], provides: [] }, }, return: { provides: [ - 'response.chat.responses', - 'response.chat.responses.streamedUsage', + 'response.chat.openaiResponses', + 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', ], @@ -715,7 +715,7 @@ export const hydrateStoredItems = ( const store = use.gateway.store; // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to. - const asked = facts['request.chat.responses'] as CanonicalResponsesPayload; + const asked = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; let expanded: CanonicalResponsesPayload; try { @@ -739,7 +739,7 @@ export const hydrateStoredItems = ( try { hydrated = hydrateResponsesPayload(expanded, store); } catch (error) { - const failure = tryCatchChatServeFailure(error); + const failure = tryCatchChatServeFailure(error); if (failure?.kind !== 'item-not-found') throw error; return move({ ...facts, @@ -754,8 +754,8 @@ export const hydrateStoredItems = ( prepared(hydrated.payload); return await next({ ...facts, - 'request.chat.responses': move(hydrated.payload), - 'request.chat.responses.privatePayloads': move([...hydrated.privatePayloads]), + 'request.chat.openaiResponses': move(hydrated.payload), + 'request.chat.openaiResponses.privatePayloads': move([...hydrated.privatePayloads]), }); }, }); @@ -770,19 +770,19 @@ export const hydrateStoredItems = ( * lets an item this turn re-emits be stored with it intact. */ export const beginStoredAttempt = defineStage< - R<'request.chat.responses.privatePayloads'>, - R<'request.chat.responses.privatePayloads'>, + R<'request.chat.openaiResponses.privatePayloads'>, + R<'request.chat.openaiResponses.privatePayloads'>, Record, Record, ChatServices >({ name: 'beginStoredAttempt', through: { - request: { needs: ['request.chat.responses.privatePayloads'], consumes: [], provides: [] }, + request: { needs: ['request.chat.openaiResponses.privatePayloads'], consumes: [], provides: [] }, response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next, use) => { - use.gateway.store.beginAttempt(new Map(facts['request.chat.responses.privatePayloads'])); + use.gateway.store.beginAttempt(new Map(facts['request.chat.openaiResponses.privatePayloads'])); return await next(facts); }, }); @@ -811,7 +811,7 @@ export const beginStoredAttempt = defineStage< * would have produced for it. */ export const simulatesCompaction = (candidate: ModelCandidate, attempt: AttemptSelector): boolean => - responsesTarget.pick(candidate.model.endpoints) !== 'responses' + openaiResponsesTarget.pick(candidate.model.endpoints) !== 'responses' || attempt.flags.includes('responses-compact-shim'); /** @@ -827,8 +827,8 @@ export const simulatesCompaction = (candidate: ModelCandidate, attempt: AttemptS * simulates can be holding one of ours, and a native one is owed its own blob verbatim. */ export const expandShimCompactions = defineStage< - R<'request.chat.responses' | 'route.attempt'>, - R<'request.chat.responses'>, + R<'request.chat.openaiResponses' | 'route.attempt'>, + R<'request.chat.openaiResponses'>, Record, Record, ChatServices @@ -836,9 +836,9 @@ export const expandShimCompactions = defineStage< name: 'expandShimCompactions', through: { request: { - needs: ['request.chat.responses', 'route.attempt'], + needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], - provides: ['request.chat.responses'], + provides: ['request.chat.openaiResponses'], }, response: { needs: [], consumes: [], provides: [] }, }, @@ -848,12 +848,12 @@ export const expandShimCompactions = defineStage< } // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to. - const payload = facts['request.chat.responses'] as CanonicalResponsesPayload; + const payload = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; const expanded = expandShimCompactionItems(payload); // A turn carrying none of ours hands the same payload on, so the record shows no change // where none happened. if (expanded === payload) return await next(facts); - return await next({ ...facts, 'request.chat.responses': move(expanded) }); + return await next({ ...facts, 'request.chat.openaiResponses': move(expanded) }); }, }); @@ -861,7 +861,7 @@ export const expandShimCompactions = defineStage< * It is the whole gate, so each chain states its own: what asks for a compaction differs by * operation, and no action travels in the record for the stage to read instead. */ export type CompactionAsk = ( - facts: R<'request.chat.responses' | 'route.attempt'>, + facts: R<'request.chat.openaiResponses' | 'route.attempt'>, use: Use, ) => boolean; @@ -878,31 +878,31 @@ export type CompactionAsk = ( * What the simulation cannot reproduce is stated where it happens, at `summarizationTurnFor`. */ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< - R<'request.chat.responses' | 'route.attempt'>, - R<'request.chat.responses'>, - R<'response.chat.responses'>, - R<'response.chat.responses'>, + R<'request.chat.openaiResponses' | 'route.attempt'>, + R<'request.chat.openaiResponses'>, + R<'response.chat.openaiResponses'>, + R<'response.chat.openaiResponses'>, ChatServices >({ name: 'summarizeForCompaction', through: { request: { - needs: ['request.chat.responses', 'route.attempt'], + needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], - provides: ['request.chat.responses'], + provides: ['request.chat.openaiResponses'], }, response: { - needs: ['response.chat.responses'], + needs: ['response.chat.openaiResponses'], consumes: [], - provides: ['response.chat.responses'], + provides: ['response.chat.openaiResponses'], }, }, execute: async (facts, next, use) => { if (!asked(facts, use)) return await next(facts); - const payload = facts['request.chat.responses'] as CanonicalResponsesPayload; - const back = await next({ ...facts, 'request.chat.responses': move(summarizationTurnFor(payload)) }); + const payload = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; + const back = await next({ ...facts, 'request.chat.openaiResponses': move(summarizationTurnFor(payload)) }); - const answer = back['response.chat.responses']; + const answer = back['response.chat.openaiResponses']; // An upstream that refused is handed on as it came: the client learns the compaction // failed rather than being given a silent empty envelope. One that answered with a single // envelope compacted on its own, and an envelope is already the answer — there are no @@ -912,9 +912,9 @@ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< // The item lifecycle is the authority on what the turn closed; a Codex upstream states an // `output` on its terminal that omits the message it just closed, so both are read and // the closed items win where there are any. - const closed = new Map(); - const observed = (async function* (): AsyncIterable> { - for await (const frame of answer.frames as AsyncIterable>) { + const closed = new Map(); + const observed = (async function* (): AsyncIterable> { + for await (const frame of answer.frames as AsyncIterable>) { if (frame.type === 'event' && frame.event.type === 'response.output_item.done') { closed.set(frame.event.output_index, frame.event.item); } @@ -928,11 +928,11 @@ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< // A summarization that closed no text produced no summary, and the blob is the whole of // what the next turn inherits — so this is a candidate that did not do the job rather // than a fault that ends the request, and the fork can try another. - return { ...back, 'response.chat.responses': move({ status: 502, message: EMPTY_SUMMARY_MESSAGE }) }; + return { ...back, 'response.chat.openaiResponses': move({ status: 502, message: EMPTY_SUMMARY_MESSAGE }) }; } const synthesized = buildCompactionEnvelope(createRandomResponsesItemId('compaction'), summaryText, collected); - return { ...back, 'response.chat.responses': move({ kind: 'stream' as const, frames: syntheticEventsFromResult(synthesized) }) }; + return { ...back, 'response.chat.openaiResponses': move({ kind: 'stream' as const, frames: syntheticEventsFromResult(synthesized) }) }; }, }); @@ -946,52 +946,52 @@ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< * native Responses upstream does with it. */ const asksForCompaction: CompactionAsk = (facts, use) => - containsCompactionTrigger((facts['request.chat.responses'] as CanonicalResponsesPayload).input) + containsCompactionTrigger((facts['request.chat.openaiResponses'] as CanonicalResponsesPayload).input) && simulatesCompaction(use.resolveAttempt(facts['route.attempt']), facts['route.attempt']); -export type ResponsesServeEntry = R< - 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.responses.wantsStream' - | 'request.chat.responses' | 'serve.model' +export type OpenAIResponsesServeEntry = R< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.openaiResponses.wantsStream' + | 'request.chat.openaiResponses' | 'serve.model' >; -export type ResponsesServeExit = R< - 'response.chat.responses.rendered' | 'response.chat.responses.streamedUsage' +export type OpenAIResponsesServeExit = R< + 'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' >; -export const responsesServePipeline = ( +export const openaiResponsesServePipeline = ( payload: CanonicalResponsesPayload, // SSE is what a run is written in when nothing else claims its frames, which is every // entry over an HTTP body; the WebSocket transport says so because it writes its own. - framing: ResponsesStreamFraming = 'sse', -): Pipeline => { + framing: OpenAIResponsesStreamFraming = 'sse', +): Pipeline => { // One cell per run, written by the stage directly above the one that reads it. The // resolver takes its narrowing at assembly, so this is where the prepared payload crosses // from the membrane to the affinity walk; until the membrane has run, what the client sent // is the whole of what is known about the turn. let prepared = payload; - return compose('responsesServe', [ + return compose('openaiResponsesServe', [ emitResponses(payload, framing), writeSettlement( - handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), - handedUp => (handedUp as { 'response.chat.responses.streamedUsage'?: unknown })['response.chat.responses.streamedUsage'] !== null, + handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), + handedUp => (handedUp as { 'response.chat.openaiResponses.streamedUsage'?: unknown })['response.chat.openaiResponses.streamedUsage'] !== null, ), hydrateStoredItems(payload, hydrated => { prepared = hydrated; }), - resolveChatCandidates(responsesNarrowing(() => prepared)), + resolveChatCandidates(openaiResponsesNarrowing(() => prepared)), failover({ - failed: handedUp => isFailure((handedUp as { 'response.chat.responses'?: unknown })['response.chat.responses']), + failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), owns: [], }), - materializeAttempt('request.chat.responses'), + materializeAttempt('request.chat.openaiResponses'), beginStoredAttempt, expandShimCompactions, summarizeForCompaction(asksForCompaction), dialChatWire({ - source: 'request.chat.responses', - needs: ['request.chat.responses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.responses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], - pick: endpoints => responsesTarget.pick(endpoints), - wire: responsesWireFor, + source: 'request.chat.openaiResponses', + needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.openaiResponses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => openaiResponsesTarget.pick(endpoints), + wire: openaiResponsesWireFor, }), ]); }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts b/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts index 4ffb09e3c9..7b7978f76d 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts @@ -1,5 +1,5 @@ import type { StatefulResponsesStore } from './items/store.ts'; -import type { CanonicalResponsesPayload } from '@floway-dev/protocols/responses'; +import type { CanonicalResponsesPayload } from '@floway-dev/protocols/openai-responses'; // Thrown when a request names a `previous_response_id` that the store cannot // resolve. The stage that hydrates catches this and answers with the diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index a107467f67..8c7e272d87 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -11,7 +11,7 @@ import type { Context } from 'hono'; import { createResponsesWsSession, type StatefulResponsesStore } from './items/store.ts'; -import { responsesServePipeline, type ResponsesFacts, type ResponsesServeExit } from './pipeline.ts'; +import { openaiResponsesServePipeline, type OpenAIResponsesFacts, type OpenAIResponsesServeExit } from './pipeline.ts'; import { openRunDump } from '../../../dump/run-sink.ts'; import type { TurnDump } from '../../../dump/turn-dump.ts'; import { apiKeyFromContext, authenticateApiKey, type AuthedContext } from '../../../middleware/auth.ts'; @@ -29,8 +29,8 @@ import { SourceStreamState } from '../shared/source-stream-state.ts'; import { move, run } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { RESPONSES_MISSING_TERMINAL_MESSAGE } from '@floway-dev/protocols/responses'; -import { isResponsesTerminalEvent, type CanonicalResponsesPayload, type ClientResponsesStreamEvent, type ResponsesRequestPayload } from '@floway-dev/protocols/responses'; +import { RESPONSES_MISSING_TERMINAL_MESSAGE } from '@floway-dev/protocols/openai-responses'; +import { isResponsesTerminalEvent, type CanonicalResponsesPayload, type ClientResponsesStreamEvent, type OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; import type { ModelCandidate } from '@floway-dev/provider'; import { toInternalDebugError } from '@floway-dev/provider'; import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; @@ -39,7 +39,7 @@ interface WorkerWebSocket extends WebSocket { accept(): void; } -interface ResponsesWebSocketSocket { +interface OpenAIResponsesWebSocketSocket { readonly readyState: number; send(data: string): void; } @@ -57,21 +57,21 @@ const UTF8_ENCODER = new TextEncoder(); // https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/resources/responses/ws-base.ts#L346-L371 export const KEEP_ALIVE_EVENT_TYPE = 'floway:keep_alive'; -interface ResponsesWebSocketHandlers { - onMessage(event: { readonly data: unknown }, socket: ResponsesWebSocketSocket): void; - onClose(event: unknown, socket: ResponsesWebSocketSocket): void; - onError(event: unknown, socket: ResponsesWebSocketSocket): void; +interface OpenAIResponsesWebSocketHandlers { + onMessage(event: { readonly data: unknown }, socket: OpenAIResponsesWebSocketSocket): void; + onClose(event: unknown, socket: OpenAIResponsesWebSocketSocket): void; + onError(event: unknown, socket: OpenAIResponsesWebSocketSocket): void; } -type ResponsesWebSocketUpgradeResolver = ( +type OpenAIResponsesWebSocketUpgradeResolver = ( c: Context, - events: ResponsesWebSocketHandlers, + events: OpenAIResponsesWebSocketHandlers, ) => Response | Promise; -let _responsesWebSocketUpgradeResolver: ResponsesWebSocketUpgradeResolver | null = null; +let _responsesWebSocketUpgradeResolver: OpenAIResponsesWebSocketUpgradeResolver | null = null; export const initResponsesWebSocketUpgradeResolver = ( - resolver: ResponsesWebSocketUpgradeResolver, + resolver: OpenAIResponsesWebSocketUpgradeResolver, ): void => { _responsesWebSocketUpgradeResolver = resolver; }; @@ -87,14 +87,14 @@ declare const WebSocketPair: { // `response.create`; the nested `response` envelope of Realtime-style clients // is an extension we also accept, and prefer when present. // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L99-L115 -type ResponsesWebSocketClientEvent = Partial & { +type OpenAIResponsesWebSocketClientEvent = Partial & { type: string; event_id?: string; - response?: Partial; + response?: Partial; [key: string]: unknown; }; -export const responsesWebSocket = async (c: AuthedContext): Promise => { +export const openaiResponsesWebSocket = async (c: AuthedContext): Promise => { if (c.req.header('upgrade')?.toLowerCase() !== 'websocket') { return Response.json({ error: 'Expected Upgrade: websocket' }, { status: 426 }); } @@ -116,7 +116,7 @@ export const responsesWebSocket = async (c: AuthedContext): Promise => return new Response(null, { status: 101, webSocket: client } as ResponseInit & { readonly webSocket: WebSocket }); }; -const createResponsesWebSocketEvents = (c: AuthedContext): ResponsesWebSocketHandlers => { +const createResponsesWebSocketEvents = (c: AuthedContext): OpenAIResponsesWebSocketHandlers => { // The upgrade authenticates the connection, but every response.create is a // separate data-plane request. Codex deliberately reuses one socket across // turns, so retain the presented credential and resolve it again before each @@ -200,7 +200,7 @@ const createResponsesWebSocketEvents = (c: AuthedContext): ResponsesWebSocketHan }; }; -interface ResponsesWsTurnFailure { +interface OpenAIResponsesWsTurnFailure { evict(): void; fail(status: number, error: Record): void; } @@ -267,7 +267,7 @@ const openResponsesWebSocketTurn = ( const handleClientMessage = async ( c: AuthedContext, - socket: ResponsesWebSocketSocket, + socket: OpenAIResponsesWebSocketSocket, session: ReturnType, data: unknown, authenticatedRawKey: string, @@ -291,7 +291,7 @@ const handleClientMessage = async ( // envelope, including the refusals the chain hands back as a body rather than a stream; the // streaming loop's `finally` covers the turn that ends failed without one. A throw inside // the loop is the single path that takes both, and `Map.delete` makes the second call inert. - const turnFailure: ResponsesWsTurnFailure = { + const turnFailure: OpenAIResponsesWsTurnFailure = { evict: () => { if (ctx === undefined || previousResponseId === undefined) return; session.evictSnapshot(ctx.store.apiKeyId, previousResponseId); @@ -337,7 +337,7 @@ const handleClientMessage = async ( const source = message.response && typeof message.response === 'object' ? message.response : Object.fromEntries(Object.entries(message).filter(([key]) => key !== 'type' && key !== 'event_id')); - const payload = responsesPayloadFromClientSource(source); + const payload = openaiResponsesPayloadFromClientSource(source); previousResponseId = payload.previous_response_id ?? undefined; const prologue = openResponsesWebSocketTurn(c, { @@ -356,13 +356,13 @@ const handleClientMessage = async ( const { facts, drain } = await run( // The transport frames its own answer, so the edge hands up the events rather than the // SSE a body would have been written from. - responsesServePipeline(payload, 'events'), + openaiResponsesServePipeline(payload, 'events'), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'responses', // A turn on this transport always streams, whatever the client wrote. - 'ingress.chat.responses.wantsStream': true, - 'request.chat.responses': payload, + 'ingress.chat.openaiResponses.wantsStream': true, + 'request.chat.openaiResponses': payload, 'serve.model': payload.model, }) as never, prologue.services as never, @@ -409,38 +409,38 @@ const wsDataToBytes = (data: unknown): Uint8Array => { throw new WebSocketClientMessageError(`Unsupported WebSocket message data: ${typeof data}`); }; -const validateClientMessage = (parsed: unknown): ResponsesWebSocketClientEvent => { +const validateClientMessage = (parsed: unknown): OpenAIResponsesWebSocketClientEvent => { if (!parsed || typeof parsed !== 'object' || typeof (parsed as { type?: unknown }).type !== 'string') { throw new WebSocketClientMessageError('WebSocket message must be a JSON object with a string type.'); } - return parsed as ResponsesWebSocketClientEvent; + return parsed as OpenAIResponsesWebSocketClientEvent; }; // The transport always streams, whatever the client sent. -const responsesPayloadFromClientSource = (source: object): CanonicalResponsesPayload => - ({ ...canonicalizeResponsesPayload(source as ResponsesRequestPayload), stream: true }); +const openaiResponsesPayloadFromClientSource = (source: object): CanonicalResponsesPayload => + ({ ...canonicalizeResponsesPayload(source as OpenAIResponsesRequestPayload), stream: true }); /** The chain hands its answer up at one key, and this entry assembled it to hand up events * rather than the SSE frames an HTTP body is written from — so what is not a stream at that * key is the one object arm this transport can see. */ const isRenderedStream = ( - rendered: ResponsesFacts['response.chat.responses.rendered'], + rendered: OpenAIResponsesFacts['response.chat.openaiResponses.rendered'], ): rendered is AsyncIterable> => Symbol.asyncIterator in rendered; const respondResponsesWebSocket = async (input: { - readonly socket: ResponsesWebSocketSocket; + readonly socket: OpenAIResponsesWebSocketSocket; readonly eventId: string | undefined; readonly signal: AbortSignal; readonly isClosed: () => boolean; readonly prologue: ChatPrologue; - readonly facts: ResponsesServeExit; + readonly facts: OpenAIResponsesServeExit; readonly drain: () => Promise; - readonly turnFailure: ResponsesWsTurnFailure; + readonly turnFailure: OpenAIResponsesWsTurnFailure; }): Promise => { const { socket, eventId, signal, isClosed, prologue, facts, drain, turnFailure } = input; const ctx = prologue.gateway; const status = facts['response.http.status']; - const rendered = facts['response.chat.responses.rendered']; + const rendered = facts['response.chat.openaiResponses.rendered']; if (!isRenderedStream(rendered)) { // A body at the rendered key is a refusal — the upstream's own, or one the chain made @@ -625,7 +625,7 @@ const respondResponsesWebSocket = async (input: { // What the turn billed, as the chain read it off the upstream's own events while they // passed. The answer is already on the socket by now, so it is settled here rather than // handed to the background the way an entry that returns a response has to. - const streamedUsage = facts['response.chat.responses.streamedUsage']; + const streamedUsage = facts['response.chat.openaiResponses.streamedUsage']; if (streamedUsage !== null) { const outcome = await streamedUsage; settleBillable({ ...prologue.services, log: consoleLogSink }, outcome.billable, outcome.failed); @@ -763,7 +763,7 @@ const normalizeErrorBody = (body: unknown, status: number): Record, eventId?: string, @@ -781,14 +781,14 @@ const sendError = ( // extension the union deliberately does not model. Both keep the untyped // `sendJson`. const sendResponsesEvent = ( - socket: ResponsesWebSocketSocket, + socket: OpenAIResponsesWebSocketSocket, event: ClientResponsesStreamEvent, eventId?: string, dump?: TurnDump | null, ): boolean => sendJson(socket, event, eventId, dump); const sendJson = ( - socket: ResponsesWebSocketSocket, + socket: OpenAIResponsesWebSocketSocket, value: unknown, eventId?: string, dump?: TurnDump | null, diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts index f1bfd7909c..1094281bfc 100644 --- a/packages/gateway/src/data-plane/chat/prologue.ts +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -13,7 +13,7 @@ import type { ChatServices } from './stages.ts'; import type { ApiKey } from '../../repo/types.ts'; import type { AttemptSelector } from '../pipeline/facts.ts'; import { gatewayCtxOptions, prologueFor, runDumpOf, type Ingress, type Prologue } from '../pipeline/serve.ts'; -import type { StatefulResponsesStore } from './responses/items/store.ts'; +import type { StatefulResponsesStore } from './openai-responses/items/store.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import type { ModelCandidate } from '@floway-dev/provider'; From c6c3b73b8c704761879fc2cb9ebcf2c476bb199c Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 05:51:50 +0800 Subject: [PATCH 079/156] refactor(chat): finish the rename the merge could not reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every remaining chat symbol takes its vendor: `ChatTargetApi` and `ModelEndpoints` values, the `ChatSourceProtocol` arm the prologue writes, and the non-chat pipeline names the assembly test still spelled short. The merge resurrected eleven modules the chat pipeline had already deleted — three `attempt.ts`, `gemini-generate-content/{serve,respond}.ts`, its interceptor directory, and two `interceptors/types.ts` — because the rename branch had modified them. Nothing imports them but each other, and the modules they import are gone, so they go. `pnpm exec tsc --noEmit -p packages/gateway` is clean. --- .../data-plane/chat-interceptors_test.ts | 234 +++++++++--------- .../chat/anthropic-messages/errors_test.ts | 12 +- .../chat/anthropic-messages/http_test.ts | 50 ++-- .../chat/chatcompletions-pipeline_test.ts | 12 +- .../chat/gemini-count-tokens_test.ts | 10 +- .../data-plane/chat/gemini-pipeline_test.ts | 14 +- .../chat/messages-count-tokens_test.ts | 10 +- .../data-plane/chat/messages-pipeline_test.ts | 14 +- .../openai-responses/compact-shim_test.ts | 8 +- .../chat/openai-responses/http_test.ts | 8 +- .../chat/openai-responses/run-dump_test.ts | 8 +- .../chat/openai-responses/websocket_test.ts | 120 ++++----- .../data-plane/chat/responses-compact_test.ts | 52 ++-- .../chat/responses-pipeline_test.ts | 78 +++--- .../data-plane/chat/responses-routes_test.ts | 4 +- .../chat/routes-chat-completions_test.ts | 6 +- .../data-plane/chat/routes-gemini_test.ts | 6 +- .../data-plane/chat/routes-messages_test.ts | 6 +- .../data-plane/chat/run-dump-frames_test.ts | 6 +- .../data-plane/chat/translated-wires_test.ts | 156 ++++++------ .../chat/translation-refusal_test.ts | 6 +- .../data-plane/pipeline/assembly_test.ts | 38 +-- .../chat/anthropic-messages/attempt.ts | 121 --------- .../chat/anthropic-messages/count-tokens.ts | 56 ++--- .../chat/anthropic-messages/errors.ts | 2 +- .../chat/anthropic-messages/http.ts | 16 +- .../chat/anthropic-messages/pipeline.ts | 82 +++--- packages/gateway/src/data-plane/chat/facts.ts | 2 +- .../chat/gemini-generate-content/attempt.ts | 170 ------------- .../gemini-generate-content/count-tokens.ts | 46 ++-- .../chat/gemini-generate-content/errors.ts | 2 +- .../chat/gemini-generate-content/http.ts | 32 +-- .../interceptors/index.ts | 29 --- .../interceptors/strip-safety-settings.ts | 11 - .../strip-unsupported-part-fields.ts | 29 --- .../interceptors/types.ts | 23 -- .../chat/gemini-generate-content/pipeline.ts | 76 +++--- .../chat/gemini-generate-content/respond.ts | 215 ---------------- .../chat/gemini-generate-content/serve.ts | 84 ------- .../src/data-plane/chat/interceptors.ts | 80 +++--- .../chat/openai-chat-completions/attempt.ts | 79 ------ .../chat/openai-chat-completions/http.ts | 6 +- .../interceptors/types.ts | 13 - .../chat/openai-chat-completions/pipeline.ts | 98 ++++---- .../chat/openai-responses/client-output.ts | 26 +- .../chat/openai-responses/compact-shim.ts | 4 +- .../chat/openai-responses/compact.ts | 68 ++--- .../data-plane/chat/openai-responses/http.ts | 18 +- .../interceptors/vendor-deepseek-normalize.ts | 44 ---- .../chat/openai-responses/items/store.ts | 176 ++++++------- .../chat/openai-responses/pipeline.ts | 166 ++++++------- .../chat/openai-responses/serve-prep.ts | 10 +- .../chat/openai-responses/websocket.ts | 70 +++--- .../gateway/src/data-plane/chat/prologue.ts | 4 +- .../interceptors/anthropic-messages/index.ts | 14 +- 55 files changed, 956 insertions(+), 1774 deletions(-) delete mode 100644 packages/gateway/src/data-plane/chat/anthropic-messages/attempt.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini-generate-content/attempt.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/index.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-safety-settings.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/types.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini-generate-content/respond.ts delete mode 100644 packages/gateway/src/data-plane/chat/gemini-generate-content/serve.ts delete mode 100644 packages/gateway/src/data-plane/chat/openai-chat-completions/attempt.ts delete mode 100644 packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/types.ts delete mode 100644 packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize.ts diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index 72b00a4a7d..a0a39a33c7 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -8,28 +8,28 @@ import { describe, expect, it } from 'vitest'; import { - applyRoleCompatibilityToChatCompletions, - applyRoleCompatibilityToMessages, - applyRoleCompatibilityToResponses, - disableReasoningOnForcedToolChoiceForChatCompletions, - disableReasoningOnForcedToolChoiceForMessages, - disableReasoningOnForcedToolChoiceForResponses, - includeUsageStreamOptionsForChatCompletions, - normalizeExclusiveCachedTokensForChatCompletions, - normalizeExclusiveCachedTokensForResponses, - normalizeUsageForChatCompletions, - stripBillingAttributionFromMessages, - stripPromptCacheKeyForChatCompletions, - stripPromptCacheKeyForResponses, - stripSafetySettingsFromGemini, - stripUnsupportedPartFieldsFromGemini, - stripUnsupportedToolsFromGemini, - suppressThoughtPartsFromGemini, - vendorDeepSeekNormalizeForChatCompletions, - vendorDeepSeekNormalizeForResponses, - vendorKimiNormalizeForChatCompletions, - vendorQwenNormalizeForChatCompletions, - vendorQwenNormalizeForResponses, + applyRoleCompatibilityToOpenAIChatCompletions, + applyRoleCompatibilityToAnthropicMessages, + applyRoleCompatibilityToOpenAIResponses, + disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + disableReasoningOnForcedToolChoiceForOpenAIResponses, + includeUsageStreamOptionsForOpenAIChatCompletions, + normalizeExclusiveCachedTokensForOpenAIChatCompletions, + normalizeExclusiveCachedTokensForOpenAIResponses, + normalizeUsageForOpenAIChatCompletions, + stripBillingAttributionFromAnthropicMessages, + stripPromptCacheKeyForOpenAIChatCompletions, + stripPromptCacheKeyForOpenAIResponses, + stripSafetySettingsFromGeminiGenerateContent, + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, + suppressThoughtPartsFromGeminiGenerateContent, + vendorDeepSeekNormalizeForOpenAIChatCompletions, + vendorDeepSeekNormalizeForOpenAIResponses, + vendorKimiNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIResponses, } from '../../src/data-plane/chat/interceptors.ts'; import type { AttemptSelector } from '../../src/data-plane/pipeline/facts.ts'; import { compose, defineStage, move, run } from '@floway-dev/pipeline'; @@ -122,7 +122,7 @@ const openaiResponsesPayload = (input: unknown[], rest: Record describe('the Chat Completions interceptors, as stages', () => { it('rewrites the roles its flags name, in the settled order', async () => { - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ { role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }, @@ -135,7 +135,7 @@ describe('the Chat Completions interceptors, as stages', () => { }); it('turns only a mid-conversation system message into a user one', async () => { - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ { role: 'system', content: 'lead' }, { role: 'system', content: 'still lead' }, @@ -153,7 +153,7 @@ describe('the Chat Completions interceptors, as stages', () => { // the conversation and the dump stops being affordable. it('hands the same payload back by identity when no flag fires', async () => { const original = openaiChatCompletionsPayload([{ role: 'system', content: 'lead' }]); - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith(), }); @@ -162,7 +162,7 @@ describe('the Chat Completions interceptors, as stages', () => { it('hands the same payload back when a flag fires but changes nothing', async () => { const original = openaiChatCompletionsPayload([{ role: 'user', content: 'hi' }]); - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('rewrite-system-to-developer'), }); @@ -174,7 +174,7 @@ describe('the Chat Completions interceptors, as stages', () => { index === 0 ? { role: 'system' as const, content: 'lead' } : { role: 'user' as const, content: `turn ${index}` } )); const original = openaiChatCompletionsPayload(messages); - const { down } = await runStage(applyRoleCompatibilityToChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('rewrite-system-to-developer'), }); @@ -184,14 +184,14 @@ describe('the Chat Completions interceptors, as stages', () => { }); it('disables reasoning only when a tool choice is forced', async () => { - const { down: forced } = await runStage(disableReasoningOnForcedToolChoiceForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down: forced } = await runStage(disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { tool_choice: 'required' }), 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); expect((forced['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).reasoning_effort).toBe('none'); const free = openaiChatCompletionsPayload([], { tool_choice: 'auto' }); - const { down: untouched } = await runStage(disableReasoningOnForcedToolChoiceForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down: untouched } = await runStage(disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': free, 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); @@ -199,7 +199,7 @@ describe('the Chat Completions interceptors, as stages', () => { }); it('removes the prompt cache key, and the key itself rather than its value', async () => { - const { down } = await runStage(stripPromptCacheKeyForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(stripPromptCacheKeyForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { prompt_cache_key: 'k' } as Partial), 'route.attempt': attemptWith('strip-prompt-cache-key'), }); @@ -210,7 +210,7 @@ describe('the Chat Completions interceptors, as stages', () => { it('leaves a payload that never had the key alone, by identity', async () => { const original = openaiChatCompletionsPayload([]); - const { down } = await runStage(stripPromptCacheKeyForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(stripPromptCacheKeyForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('strip-prompt-cache-key'), }); @@ -220,14 +220,14 @@ describe('the Chat Completions interceptors, as stages', () => { // The usage chunk is what every stream here is billed from, so what the client asked for and // what the upstream is asked for differ — and this is the rule that makes them differ. it('asks the upstream for the usage chunk when the client sent no stream options', async () => { - const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(includeUsageStreamOptionsForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), }); expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); }); it('asks for it over a client that asked for it to be left out', async () => { - const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(includeUsageStreamOptionsForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { stream_options: { include_usage: false } }), }); expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); @@ -236,7 +236,7 @@ describe('the Chat Completions interceptors, as stages', () => { // Everything else on `stream_options` is the client's own. An upstream that reads a key our // typed surface does not name still gets it. it('keeps the client-s other stream options while it forces the one it wants', async () => { - const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(includeUsageStreamOptionsForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { stream_options: { extra: 'keep-me', include_usage: false } as unknown as OpenAIChatCompletionsPayload['stream_options'], }), @@ -247,7 +247,7 @@ describe('the Chat Completions interceptors, as stages', () => { it('leaves a payload that already asked for it alone, by identity', async () => { const original = openaiChatCompletionsPayload([], { stream_options: { include_usage: true } }); - const { down } = await runStage(includeUsageStreamOptionsForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(includeUsageStreamOptionsForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': original, }); expect(down['request.chat.openaiChatCompletions']).toBe(original); @@ -256,7 +256,7 @@ describe('the Chat Completions interceptors, as stages', () => { // The spec puts the final usage on a carrier chunk of its own; an upstream that hangs it off // the last delta is split in two so that everything downstream reads the one shape. it('moves usage off the chunk that carried the last delta and onto a carrier of its own', async () => { - const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.openaiChatCompletions', {}, + const { up } = await runStage(normalizeUsageForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', {}, streamOf([eventFrame({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], @@ -285,7 +285,7 @@ describe('the Chat Completions interceptors, as stages', () => { } as unknown as OpenAIChatCompletionsStreamEvent); const done = doneFrame(); - const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.openaiChatCompletions', {}, + const { up } = await runStage(normalizeUsageForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', {}, streamOf([carrier, content, done])); const frames = await drain(up['response.chat.openaiChatCompletions']); @@ -295,7 +295,7 @@ describe('the Chat Completions interceptors, as stages', () => { // Naming a vendor's cache fields is the vendor stage's job, and it has already run by the // time the answer reaches this one. A chunk still carrying them is one no vendor claimed. it('leaves a vendor-s own cache fields where it found them', async () => { - const { up } = await runStage(normalizeUsageForChatCompletions, 'response.chat.openaiChatCompletions', {}, + const { up } = await runStage(normalizeUsageForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', {}, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 25, })])); @@ -310,7 +310,7 @@ describe('the Chat Completions interceptors, as stages', () => { // counts only what the cache did not serve, and `total_tokens` witnesses it whatever anyone // flagged. it('folds the cache buckets into the input total when the totals witness it', async () => { - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith(), }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_tokens_details: { cached_tokens: 13312 }, @@ -329,7 +329,7 @@ describe('the Chat Completions interceptors, as stages', () => { // input rather than the gate — and it arrives on the attempt this stage read on the way // down, so this is also what proves the closure carries it to the way back. it('folds on the flag alone when the chunk states no total, cache writes included', async () => { - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 10, prompt_tokens_details: { cached_tokens: 120, cache_creation_input_tokens: 80 }, @@ -345,12 +345,12 @@ describe('the Chat Completions interceptors, as stages', () => { }); const uncached = openaiChatCompletionsUsageChunk({ prompt_tokens: 40, completion_tokens: 1, total_tokens: 41 }); - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), }, streamOf([uncached])); expect(await drain(up['response.chat.openaiChatCompletions'])).toEqual([uncached]); - const { up: unflagged } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up: unflagged } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith(), }, streamOf([inclusive])); expect(await drain(unflagged['response.chat.openaiChatCompletions'])).toEqual([inclusive]); @@ -360,7 +360,7 @@ describe('the Chat Completions interceptors, as stages', () => { // operator makes on that upstream. Neither may pass silently: one over-charges the input by // the whole cached prefix, the other underflows it. it('raises when the flag claims exclusive and the totals say inclusive', async () => { - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_tokens_details: { cached_tokens: 400 }, @@ -371,7 +371,7 @@ describe('the Chat Completions interceptors, as stages', () => { }); it('raises naming the flag when the cache counts underflow with no verdict', async () => { - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith(), }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 479, completion_tokens: 373, prompt_tokens_details: { cached_tokens: 13312 }, @@ -383,7 +383,7 @@ describe('the Chat Completions interceptors, as stages', () => { it('leaves an answer that is not a stream alone, by identity', async () => { const answer = { kind: 'value' as const, body: null }; - const { up } = await runStage(normalizeExclusiveCachedTokensForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith(), }, answer); expect(up['response.chat.openaiChatCompletions']).toBe(answer); @@ -428,7 +428,7 @@ const assistantOf = (payload: unknown): Record => describe('the Chat Completions vendor dialects, as stages', () => { it('projects the canonical assistant reasoning onto the one field DeepSeek reads', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload(assistantReplay({ reasoning_text: 'let me check the docs', reasoning_opaque: 'opaque-blob', @@ -448,7 +448,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('writes that field from the summary items when the scalar is the thing that is missing', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload(assistantReplay({ reasoning_items: [{ type: 'reasoning', id: 'rs_1', @@ -462,7 +462,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('strips the fields it cannot project even when there is nothing to project', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ { role: 'user', content: 'first turn' }, { role: 'assistant', content: 'answer', reasoning_items: [{ type: 'reasoning' }], reasoning_opaque: 'opaque-chain' }, @@ -478,7 +478,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('puts the reasoning sentinel into the shape DeepSeek reads', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { reasoning_effort: 'none' }), 'route.attempt': attemptWith('vendor-deepseek'), }); @@ -489,7 +489,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('puts the reasoning sentinel into the shape Qwen reads', async () => { - const { down } = await runStage(vendorQwenNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(vendorQwenNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { reasoning_effort: 'none' }), 'route.attempt': attemptWith('vendor-qwen'), }); @@ -503,7 +503,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { // and DeepSeek and Qwen both take it in the canonical field. it('leaves a payload that asked for real reasoning alone, by identity', async () => { const original = openaiChatCompletionsPayload([], { reasoning_effort: 'high' }); - for (const stage of [vendorDeepSeekNormalizeForChatCompletions, vendorQwenNormalizeForChatCompletions]) { + for (const stage of [vendorDeepSeekNormalizeForOpenAIChatCompletions, vendorQwenNormalizeForOpenAIChatCompletions]) { const { down } = await runStage(stage, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('vendor-deepseek', 'vendor-qwen'), @@ -515,7 +515,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { // DeepSeek's structured output takes `json_object` and nothing else, so the schema goes // rather than the request being refused for carrying it. it('downgrades a JSON schema to the JSON object DeepSeek supports', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { response_format: { type: 'json_schema', @@ -530,7 +530,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { it('leaves a payload already asking for a JSON object alone, by identity', async () => { const original = openaiChatCompletionsPayload([], { response_format: { type: 'json_object' } }); - const { down } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': original, 'route.attempt': attemptWith('vendor-deepseek'), }); @@ -538,7 +538,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('renames DeepSeek-s reasoning deltas to the canonical field', async () => { - const { up } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), 'route.attempt': attemptWith('vendor-deepseek'), }, streamOf([ @@ -557,7 +557,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('rewrites DeepSeek-s cache-hit count into the field the rules above it read', async () => { - const { up } = await runStage(vendorDeepSeekNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), 'route.attempt': attemptWith('vendor-deepseek'), }, streamOf([openaiChatCompletionsUsageChunk({ @@ -576,7 +576,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); it('rewrites Kimi-s flat cached count into that same field', async () => { - const { up } = await runStage(vendorKimiNormalizeForChatCompletions, 'response.chat.openaiChatCompletions', { + const { up } = await runStage(vendorKimiNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'route.attempt': attemptWith('vendor-kimi'), }, streamOf([openaiChatCompletionsUsageChunk({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, cached_tokens: 50, @@ -594,8 +594,8 @@ describe('the Chat Completions vendor dialects, as stages', () => { // array's own indices spread across it. it('replaces a details field that is not an object rather than spreading it', async () => { for (const [stage, flag, vendorField] of [ - [vendorDeepSeekNormalizeForChatCompletions, 'vendor-deepseek', 'prompt_cache_hit_tokens'], - [vendorKimiNormalizeForChatCompletions, 'vendor-kimi', 'cached_tokens'], + [vendorDeepSeekNormalizeForOpenAIChatCompletions, 'vendor-deepseek', 'prompt_cache_hit_tokens'], + [vendorKimiNormalizeForOpenAIChatCompletions, 'vendor-kimi', 'cached_tokens'], ] as const) { const { up } = await runStage(stage, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), @@ -618,9 +618,9 @@ describe('the Chat Completions vendor dialects, as stages', () => { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 50, }); for (const stage of [ - vendorDeepSeekNormalizeForChatCompletions, - vendorQwenNormalizeForChatCompletions, - vendorKimiNormalizeForChatCompletions, + vendorDeepSeekNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIChatCompletions, + vendorKimiNormalizeForOpenAIChatCompletions, ]) { const { down, up } = await runStage(stage, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': payload, @@ -637,8 +637,8 @@ describe('the Chat Completions vendor dialects, as stages', () => { it('lets an upstream stream failure through as it was thrown', async () => { const failure = new Error('upstream stream failed'); for (const [stage, flag] of [ - [vendorDeepSeekNormalizeForChatCompletions, 'vendor-deepseek'], - [vendorKimiNormalizeForChatCompletions, 'vendor-kimi'], + [vendorDeepSeekNormalizeForOpenAIChatCompletions, 'vendor-deepseek'], + [vendorKimiNormalizeForOpenAIChatCompletions, 'vendor-kimi'], ] as const) { const { up } = await runStage(stage, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), @@ -652,7 +652,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { describe('the Messages interceptors, as stages', () => { it('turns every inline system message into a user one', async () => { - const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(applyRoleCompatibilityToAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': anthropicMessagesPayload([ { role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }, @@ -668,7 +668,7 @@ describe('the Messages interceptors, as stages', () => { it('hands the same conversation back by identity when the flag does not fire', async () => { const original = anthropicMessagesPayload([{ role: 'system', content: 'lead' }]); - const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(applyRoleCompatibilityToAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': original, 'route.attempt': attemptWith(), }); @@ -677,7 +677,7 @@ describe('the Messages interceptors, as stages', () => { it('hands the same conversation back when the flag fires but there is no system message', async () => { const original = anthropicMessagesPayload([{ role: 'user', content: 'hi' }]); - const { down } = await runStage(applyRoleCompatibilityToMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(applyRoleCompatibilityToAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': original, 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), }); @@ -685,7 +685,7 @@ describe('the Messages interceptors, as stages', () => { }); it('disables thinking on a forced tool choice and leaves the structured-output format', async () => { - const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': anthropicMessagesPayload([], { tool_choice: { type: 'any' }, output_config: { effort: 'high', format: { type: 'json_schema', schema: {} } }, @@ -700,7 +700,7 @@ describe('the Messages interceptors, as stages', () => { }); it('drops an output_config that held nothing but the effort', async () => { - const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': anthropicMessagesPayload([], { tool_choice: { type: 'tool', name: 'f' }, output_config: { effort: 'high' }, @@ -712,7 +712,7 @@ describe('the Messages interceptors, as stages', () => { it('leaves a free tool choice alone, by identity', async () => { const free = anthropicMessagesPayload([], { tool_choice: { type: 'auto' } }); - const { down } = await runStage(disableReasoningOnForcedToolChoiceForMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': free, 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); @@ -720,7 +720,7 @@ describe('the Messages interceptors, as stages', () => { }); it('scrubs the billing header line and the per-turn hash out of the system prompt', async () => { - const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(stripBillingAttributionFromAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': anthropicMessagesPayload([], { system: 'x-anthropic-billing-header: cch=abcdef12\nreal instructions\ncch=beef01;', }), @@ -730,7 +730,7 @@ describe('the Messages interceptors, as stages', () => { }); it('drops a system block the scrub emptied, and the field when nothing is left', async () => { - const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(stripBillingAttributionFromAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': anthropicMessagesPayload([], { system: [{ type: 'text', text: 'x-anthropic-billing-header: cch=abcdef12' }], }), @@ -743,7 +743,7 @@ describe('the Messages interceptors, as stages', () => { it('leaves a system prompt carrying no attribution block alone, by identity', async () => { const original = anthropicMessagesPayload([], { system: [{ type: 'text', text: 'real instructions' }] }); - const { down } = await runStage(stripBillingAttributionFromMessages, 'response.chat.anthropicMessages', { + const { down } = await runStage(stripBillingAttributionFromAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': original, 'route.attempt': attemptWith('strip-billing-attribution'), }); @@ -753,7 +753,7 @@ describe('the Messages interceptors, as stages', () => { describe('the Gemini interceptors, as stages', () => { it('drops the part fields no translation carries, and a part they were all of', async () => { - const { down } = await runStage(stripUnsupportedPartFieldsFromGemini, 'response.chat.geminiGenerateContent', { + const { down } = await runStage(stripUnsupportedPartFieldsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': { contents: [{ role: 'user', @@ -772,14 +772,14 @@ describe('the Gemini interceptors, as stages', () => { it('hands a payload carrying no unsupported part field back by identity', async () => { const original: GeminiGenerateContentPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; - const { down } = await runStage(stripUnsupportedPartFieldsFromGemini, 'response.chat.geminiGenerateContent', { + const { down } = await runStage(stripUnsupportedPartFieldsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': original, }); expect(down['request.chat.geminiGenerateContent']).toBe(original); }); it('keeps a tool group only for its function declarations', async () => { - const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.geminiGenerateContent', { + const { down } = await runStage(stripUnsupportedToolsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': { tools: [{ functionDeclarations: [{ name: 'f' }], googleSearch: {} }, { codeExecution: {} }], } satisfies GeminiGenerateContentPayload, @@ -788,7 +788,7 @@ describe('the Gemini interceptors, as stages', () => { }); it('removes the tool list itself when no group declared a function', async () => { - const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.geminiGenerateContent', { + const { down } = await runStage(stripUnsupportedToolsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': { tools: [{ googleSearch: {} }] } satisfies GeminiGenerateContentPayload, }); // An empty tool list is a different request from no tools at all. @@ -797,20 +797,20 @@ describe('the Gemini interceptors, as stages', () => { it('hands a payload whose groups only declare functions back by identity', async () => { const original: GeminiGenerateContentPayload = { tools: [{ functionDeclarations: [{ name: 'f' }] }] }; - const { down } = await runStage(stripUnsupportedToolsFromGemini, 'response.chat.geminiGenerateContent', { + const { down } = await runStage(stripUnsupportedToolsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': original, }); expect(down['request.chat.geminiGenerateContent']).toBe(original); }); it('removes the safety settings, and leaves a payload that carries none alone', async () => { - const { down } = await runStage(stripSafetySettingsFromGemini, 'response.chat.geminiGenerateContent', { + const { down } = await runStage(stripSafetySettingsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': { safetySettings: [{ category: 'c', threshold: 't' }] } satisfies GeminiGenerateContentPayload, }); expect('safetySettings' in (down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload)).toBe(false); const original: GeminiGenerateContentPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; - const { down: untouched } = await runStage(stripSafetySettingsFromGemini, 'response.chat.geminiGenerateContent', { + const { down: untouched } = await runStage(stripSafetySettingsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': original, }); expect(untouched['request.chat.geminiGenerateContent']).toBe(original); @@ -827,7 +827,7 @@ describe('the Gemini interceptors, as stages', () => { }], }), ]); - const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.geminiGenerateContent', { + const { up } = await runStage(suppressThoughtPartsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': {} satisfies GeminiGenerateContentPayload, }, answer); const frames = await drain(up['response.chat.geminiGenerateContent']); @@ -844,7 +844,7 @@ describe('the Gemini interceptors, as stages', () => { const answer = streamOf([ eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }] }, finishReason: 'STOP' }] }), ]); - const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.geminiGenerateContent', { + const { up } = await runStage(suppressThoughtPartsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': { generationConfig: { thinkingConfig: { includeThoughts: true } } } satisfies GeminiGenerateContentPayload, }, answer); expect(up['response.chat.geminiGenerateContent']).toBe(answer); @@ -852,7 +852,7 @@ describe('the Gemini interceptors, as stages', () => { it('leaves an answer that is not a stream alone, by identity', async () => { const answer = { kind: 'value' as const, body: null }; - const { up } = await runStage(suppressThoughtPartsFromGemini, 'response.chat.geminiGenerateContent', { + const { up } = await runStage(suppressThoughtPartsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': {} satisfies GeminiGenerateContentPayload, }, answer); expect(up['response.chat.geminiGenerateContent']).toBe(answer); @@ -865,7 +865,7 @@ describe('the Responses interceptors, as stages', () => { .map(item => (item as { role?: unknown }).role); it('rewrites the roles its flags name, in the settled order', async () => { - const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(applyRoleCompatibilityToOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': openaiResponsesPayload([ { type: 'message', role: 'system', content: 'lead' }, { type: 'message', role: 'user', content: 'hi' }, @@ -879,7 +879,7 @@ describe('the Responses interceptors, as stages', () => { // The ordering the rule depends on: an item with no role of its own still ends the leading // system run, which is what makes the system message after it mid-conversation. it('lets an item carrying no role cross the leading system run', async () => { - const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(applyRoleCompatibilityToOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': openaiResponsesPayload([ { type: 'message', role: 'system', content: 'lead' }, { type: 'reasoning', id: 'rs_1', summary: [] }, @@ -892,7 +892,7 @@ describe('the Responses interceptors, as stages', () => { it('hands the same payload back by identity when no flag fires', async () => { const original = openaiResponsesPayload([{ type: 'message', role: 'system', content: 'lead' }]); - const { down } = await runStage(applyRoleCompatibilityToResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(applyRoleCompatibilityToOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': original, 'route.attempt': attemptWith(), }); @@ -901,7 +901,7 @@ describe('the Responses interceptors, as stages', () => { it('disables reasoning on a required tool choice and on a named one', async () => { for (const toolChoice of ['required', { type: 'function', name: 'f' }]) { - const { down } = await runStage(disableReasoningOnForcedToolChoiceForResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': openaiResponsesPayload([], { tool_choice: toolChoice, reasoning: { effort: 'high', summary: 'auto' } }), 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); @@ -913,7 +913,7 @@ describe('the Responses interceptors, as stages', () => { it('leaves an automatic tool choice alone, by identity', async () => { const free = openaiResponsesPayload([], { tool_choice: 'auto' }); - const { down } = await runStage(disableReasoningOnForcedToolChoiceForResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': free, 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), }); @@ -921,14 +921,14 @@ describe('the Responses interceptors, as stages', () => { }); it('removes the prompt cache key, and leaves a payload that never had it alone', async () => { - const { down } = await runStage(stripPromptCacheKeyForResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(stripPromptCacheKeyForOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': openaiResponsesPayload([], { prompt_cache_key: 'k' }), 'route.attempt': attemptWith('strip-prompt-cache-key'), }); expect('prompt_cache_key' in (down['request.chat.openaiResponses'] as OpenAIResponsesPayload)).toBe(false); const original = openaiResponsesPayload([]); - const { down: untouched } = await runStage(stripPromptCacheKeyForResponses, 'response.chat.openaiResponses', { + const { down: untouched } = await runStage(stripPromptCacheKeyForOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': original, 'route.attempt': attemptWith('strip-prompt-cache-key'), }); @@ -939,7 +939,7 @@ describe('the Responses interceptors, as stages', () => { // 100 + 50 + 10 = 160 reaches the stated total and 100 + 10 does not, so the upstream // reports the buckets outside the input count whatever anyone flagged. const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); - const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.openaiResponses', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIResponses, 'response.chat.openaiResponses', { 'route.attempt': attemptWith(), }, answer); const frames = await drain(up['response.chat.openaiResponses']); @@ -951,7 +951,7 @@ describe('the Responses interceptors, as stages', () => { // down, so this is also what proves the closure carries it to the way back. it('folds on the flag alone when the response states no total', async () => { const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, input_tokens_details: { cached_tokens: 50, cache_write_tokens: 5 } })]); - const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.openaiResponses', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIResponses, 'response.chat.openaiResponses', { 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), }, answer); const frames = await drain(up['response.chat.openaiResponses']); @@ -960,7 +960,7 @@ describe('the Responses interceptors, as stages', () => { it('hands a frame back by identity when the totals say the buckets are already inside', async () => { const frame = usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 110, input_tokens_details: { cached_tokens: 50 } }); - const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.openaiResponses', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIResponses, 'response.chat.openaiResponses', { 'route.attempt': attemptWith(), }, streamOf([frame])); const frames = await drain(up['response.chat.openaiResponses']); @@ -969,14 +969,14 @@ describe('the Responses interceptors, as stages', () => { it('leaves an answer that is not a stream alone, by identity', async () => { const answer = { kind: 'value' as const, body: null }; - const { up } = await runStage(normalizeExclusiveCachedTokensForResponses, 'response.chat.openaiResponses', { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIResponses, 'response.chat.openaiResponses', { 'route.attempt': attemptWith(), }, answer); expect(up['response.chat.openaiResponses']).toBe(answer); }); it('puts the reasoning sentinel into the shape DeepSeek reads', async () => { - const { down } = await runStage(vendorDeepSeekNormalizeForResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': openaiResponsesPayload([], { reasoning: { effort: 'none' } }), 'route.attempt': attemptWith('vendor-deepseek'), }); @@ -986,7 +986,7 @@ describe('the Responses interceptors, as stages', () => { }); it('puts the reasoning sentinel into the shape Qwen reads', async () => { - const { down } = await runStage(vendorQwenNormalizeForResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(vendorQwenNormalizeForOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': openaiResponsesPayload([], { reasoning: { effort: 'none' } }), 'route.attempt': attemptWith('vendor-qwen'), }); @@ -1000,7 +1000,7 @@ describe('the Responses interceptors, as stages', () => { // the next upstream, and a rewrite that fired without the flag would put a Qwen field on a // wire that has never heard of it. const original = openaiResponsesPayload([], { reasoning: { effort: 'none' } }); - const { down } = await runStage(vendorQwenNormalizeForResponses, 'response.chat.openaiResponses', { + const { down } = await runStage(vendorQwenNormalizeForOpenAIResponses, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': original, 'route.attempt': attemptWith(), }); @@ -1009,7 +1009,7 @@ describe('the Responses interceptors, as stages', () => { it('leaves a payload that asked for real reasoning alone, by identity', async () => { const original = openaiResponsesPayload([], { reasoning: { effort: 'high' } }); - for (const stage of [vendorDeepSeekNormalizeForResponses, vendorQwenNormalizeForResponses]) { + for (const stage of [vendorDeepSeekNormalizeForOpenAIResponses, vendorQwenNormalizeForOpenAIResponses]) { const { down } = await runStage(stage, 'response.chat.openaiResponses', { 'request.chat.openaiResponses': original, 'route.attempt': attemptWith('vendor-deepseek', 'vendor-qwen'), @@ -1037,9 +1037,9 @@ const usageOf = (frame: ProtocolFrame): Record { it('assembles and runs the Messages array', async () => { const { down } = await runChain('response.chat.anthropicMessages', [ - stripBillingAttributionFromMessages, - disableReasoningOnForcedToolChoiceForMessages, - applyRoleCompatibilityToMessages, + stripBillingAttributionFromAnthropicMessages, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + applyRoleCompatibilityToAnthropicMessages, ], { 'request.chat.anthropicMessages': anthropicMessagesPayload([{ role: 'system', content: 'inline' }], { system: 'x-anthropic-billing-header: cch=abcdef12\nreal instructions', @@ -1062,10 +1062,10 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }, { text: 'the answer' }] }, finishReason: 'STOP' }] }), ]); const { down, up } = await runChain('response.chat.geminiGenerateContent', [ - stripUnsupportedPartFieldsFromGemini, - stripUnsupportedToolsFromGemini, - stripSafetySettingsFromGemini, - suppressThoughtPartsFromGemini, + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, + stripSafetySettingsFromGeminiGenerateContent, + suppressThoughtPartsFromGeminiGenerateContent, ], { 'request.chat.geminiGenerateContent': { contents: [{ role: 'user', parts: [{ text: 'hi', fileData: { mimeType: 'text/plain', fileUri: 'u' } }] }], @@ -1088,12 +1088,12 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => it('assembles and runs the Responses array, the vendor normalizers last', async () => { const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); const { down, up } = await runChain('response.chat.openaiResponses', [ - disableReasoningOnForcedToolChoiceForResponses, - applyRoleCompatibilityToResponses, - stripPromptCacheKeyForResponses, - normalizeExclusiveCachedTokensForResponses, - vendorDeepSeekNormalizeForResponses, - vendorQwenNormalizeForResponses, + disableReasoningOnForcedToolChoiceForOpenAIResponses, + applyRoleCompatibilityToOpenAIResponses, + stripPromptCacheKeyForOpenAIResponses, + normalizeExclusiveCachedTokensForOpenAIResponses, + vendorDeepSeekNormalizeForOpenAIResponses, + vendorQwenNormalizeForOpenAIResponses, ], { 'request.chat.openaiResponses': openaiResponsesPayload([ { type: 'message', role: 'system', content: 'lead' }, @@ -1133,13 +1133,13 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => }, } as unknown as OpenAIChatCompletionsStreamEvent)]); const { down, up } = await runChain('response.chat.openaiChatCompletions', [ - includeUsageStreamOptionsForChatCompletions, - normalizeUsageForChatCompletions, - applyRoleCompatibilityToChatCompletions, - normalizeExclusiveCachedTokensForChatCompletions, - vendorDeepSeekNormalizeForChatCompletions, - vendorQwenNormalizeForChatCompletions, - vendorKimiNormalizeForChatCompletions, + includeUsageStreamOptionsForOpenAIChatCompletions, + normalizeUsageForOpenAIChatCompletions, + applyRoleCompatibilityToOpenAIChatCompletions, + normalizeExclusiveCachedTokensForOpenAIChatCompletions, + vendorDeepSeekNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIChatCompletions, + vendorKimiNormalizeForOpenAIChatCompletions, ], { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ { role: 'system', content: 'lead' }, diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts index a79469aca2..f13c9af0ce 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts @@ -5,11 +5,11 @@ import { test } from 'vitest'; -import { renderMessagesError } from '../../../../src/data-plane/chat/anthropic-messages/errors.ts'; +import { renderAnthropicMessagesError } from '../../../../src/data-plane/chat/anthropic-messages/errors.ts'; import { assert, assertEquals } from '@floway-dev/test-utils'; test('renders an Anthropic invalid_request_error envelope with a top-level request_id', () => { - const body = renderMessagesError( + const body = renderAnthropicMessagesError( 400, "Invalid 'image_url' content part in system or developer message. Only 'text' content parts are supported in system messages on this model.", ); @@ -26,7 +26,7 @@ test('renders an Anthropic invalid_request_error envelope with a top-level reque }); test('preserves Anthropic key order: type, error, request_id', () => { - const raw = JSON.stringify(renderMessagesError(400, 'whatever')); + const raw = JSON.stringify(renderAnthropicMessagesError(400, 'whatever')); assert( /^\{"type":"error","error":\{"type":"invalid_request_error","message":"whatever"\},"request_id":"req_[A-Za-z0-9]{24}"\}$/.test(raw), `body ${raw} must match Anthropic-direct byte shape`, @@ -36,7 +36,7 @@ test('preserves Anthropic key order: type, error, request_id', () => { test('names the condition each status carries in this protocol', () => { // The status a refusal was made with is what selects the type name, so the mapping is what // makes a gateway-made refusal read like the upstream's own. - assertEquals((renderMessagesError(429, 'slow down').error as { type: string }).type, 'rate_limit_error'); - assertEquals((renderMessagesError(529, 'overloaded').error as { type: string }).type, 'overloaded_error'); - assertEquals((renderMessagesError(502, 'bad gateway').error as { type: string }).type, 'api_error'); + assertEquals((renderAnthropicMessagesError(429, 'slow down').error as { type: string }).type, 'rate_limit_error'); + assertEquals((renderAnthropicMessagesError(529, 'overloaded').error as { type: string }).type, 'overloaded_error'); + assertEquals((renderAnthropicMessagesError(502, 'bad gateway').error as { type: string }).type, 'api_error'); }); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index b12962a14c..1b63323603 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -75,7 +75,7 @@ const makeApp = (): Hono<{ Variables: AuthVars }> => { return app; }; -const makeMessagesEvents = (): readonly AnthropicMessagesStreamEvent[] => [ +const makeAnthropicMessagesEvents = (): readonly AnthropicMessagesStreamEvent[] => [ { type: 'message_start', message: { @@ -104,13 +104,13 @@ const makeProtocolFrames = async function* (events: readonly TEvent[]): const makeCandidate = (overrides: { upstream?: string; endpoints?: ModelEndpoints; - callMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; + callAnthropicMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; + callAnthropicMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; } = {}): ModelCandidate => { const upstream = overrides.upstream ?? 'up_test'; const provider = stubProvider({ - callMessages: overrides.callMessages, - callMessagesCountTokens: overrides.callMessagesCountTokens, + callAnthropicMessages: overrides.callAnthropicMessages, + callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, }); return { provider: { @@ -124,10 +124,10 @@ const makeCandidate = (overrides: { test('POST /v1/messages streams a successful SSE body', async () => { installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers(), + const callAnthropicMessages = vi.fn(async (): Promise> => ({ + ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: new Headers(), })); - queueCandidates([makeCandidate({ callMessages })]); + queueCandidates([makeCandidate({ callAnthropicMessages })]); const response = await makeApp().request('/v1/messages', { method: 'POST', @@ -140,15 +140,15 @@ test('POST /v1/messages streams a successful SSE body', async () => { const body = await response.text(); assert(body.includes('event: message_start')); assert(body.includes('event: message_stop')); - assertEquals(callMessages.mock.calls.length, 1); + assertEquals(callAnthropicMessages.mock.calls.length, 1); }); test('POST /v1/messages returns a single JSON body when stream is omitted', async () => { installRepo(); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: new Headers(), + const callAnthropicMessages = vi.fn(async (): Promise> => ({ + ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: new Headers(), })); - queueCandidates([makeCandidate({ callMessages })]); + queueCandidates([makeCandidate({ callAnthropicMessages })]); const response = await makeApp().request('/v1/messages', { method: 'POST', @@ -164,10 +164,10 @@ test('POST /v1/messages returns a single JSON body when stream is omitted', asyn test('POST /v1/messages answers the Claude Code model-validation probe without calling the upstream', async () => { const repo = installRepo(); - const callMessages = vi.fn((): Promise> => { + const callAnthropicMessages = vi.fn((): Promise> => { throw new Error('the probe reached the upstream'); }); - queueCandidates([makeCandidate({ callMessages })]); + queueCandidates([makeCandidate({ callAnthropicMessages })]); const response = await makeApp().request('/v1/messages', { method: 'POST', @@ -187,7 +187,7 @@ test('POST /v1/messages answers the Claude Code model-validation probe without c assertEquals(body.stop_reason, 'max_tokens'); assertEquals(body.usage.input_tokens, 0); assertEquals(body.usage.output_tokens, 0); - assertEquals(callMessages.mock.calls.length, 0); + assertEquals(callAnthropicMessages.mock.calls.length, 0); // The turn is recorded as served, at zero cost, and contributes no latency // sample — there was no upstream call to measure. @@ -221,11 +221,11 @@ test('POST /v1/messages rejects body anthropic_beta with a 400 before routing', test('POST /v1/messages/count_tokens proxies the upstream measurement body', async () => { const repo = installRepo(); - const callMessagesCountTokens = vi.fn(async (): Promise => ({ + const callAnthropicMessagesCountTokens = vi.fn(async (): Promise => ({ response: new Response(JSON.stringify({ input_tokens: 99 }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), modelKey: 'k', })); - queueCandidates([makeCandidate({ callMessagesCountTokens })]); + queueCandidates([makeCandidate({ callAnthropicMessagesCountTokens })]); const response = await makeApp().request('/v1/messages/count_tokens', { method: 'POST', @@ -236,7 +236,7 @@ test('POST /v1/messages/count_tokens proxies the upstream measurement body', asy assertEquals(response.status, 200); const body = await response.json() as { input_tokens: number }; assertEquals(body.input_tokens, 99); - assertEquals(callMessagesCountTokens.mock.calls.length, 1); + assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); // A measurement goes through settlement like every other run. It names no billed entity — // nothing here is billable today — but the row is written, because "count_tokens is exempt @@ -258,10 +258,10 @@ test('POST /v1/messages forwards upstream response headers end-to-end (streaming 'connection': 'close', 'set-cookie': 'session=secret', }); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: upstreamHeaders, + const callAnthropicMessages = vi.fn(async (): Promise> => ({ + ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: upstreamHeaders, })); - queueCandidates([makeCandidate({ callMessages })]); + queueCandidates([makeCandidate({ callAnthropicMessages })]); const response = await makeApp().request('/v1/messages', { method: 'POST', @@ -289,10 +289,10 @@ test('POST /v1/messages forwards upstream response headers end-to-end (non-strea 'anthropic-ratelimit-unified-status': 'allowed', 'cf-ray': 'cf_ray_e2e', }); - const callMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeMessagesEvents()), modelKey: 'k', headers: upstreamHeaders, + const callAnthropicMessages = vi.fn(async (): Promise> => ({ + ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: upstreamHeaders, })); - queueCandidates([makeCandidate({ callMessages })]); + queueCandidates([makeCandidate({ callAnthropicMessages })]); const response = await makeApp().request('/v1/messages', { method: 'POST', @@ -311,7 +311,7 @@ test('POST /v1/messages renders the Anthropic-shaped model-unsupported 400 when // anthropicMessagesGenerateTarget (messages > responses > chat-completions) rejects // it, leaving zero viable candidates, and with sawModel=true the serve // renders model-unsupported as a 400. - queueCandidates([makeCandidate({ endpoints: { completions: {} } })]); + queueCandidates([makeCandidate({ endpoints: { openaiCompletions: {} } })]); const response = await makeApp().request('/v1/messages', { method: 'POST', diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index d2f4a8d54a..e583096bec 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -11,7 +11,7 @@ import { enumerateModelCandidates } from '../../../src/data-plane/providers/reso import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import { CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import { OPENAI_CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { SseFrame } from '@floway-dev/protocols/common'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -24,16 +24,16 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal let live: readonly ModelCandidate[] = []; const candidate = ( - callChatCompletions: (model: unknown, body: unknown) => Promise>, + callOpenAIChatCompletions: (model: unknown, body: unknown) => Promise>, upstreamId = 'up_a', flags: readonly FlagId[] = [], ): ModelCandidate => { - const endpoints = { chatCompletions: {} }; + const endpoints = { openaiChatCompletions: {} }; return { provider: { upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callChatCompletions }), + instance: stubProvider({ callOpenAIChatCompletions }), }, model: stubInternalModel( { @@ -100,7 +100,7 @@ const serveWith = async ( openaiChatCompletionsServePipeline(payload), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], - 'ingress.chat.sourceProtocol': 'chatCompletions', + 'ingress.chat.sourceProtocol': 'openaiChatCompletions', 'ingress.chat.openaiChatCompletions.wantsStream': wantsStream, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': wantsUsageChunk, 'request.chat.openaiChatCompletions': payload, @@ -422,7 +422,7 @@ describe('the chat completions chain', () => { for await (const _frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { /* to the end */ } }; - await expect(drain()).rejects.toThrow(CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE); + await expect(drain()).rejects.toThrow(OPENAI_CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE); }); // Anything an upstream keeps sending after its terminator is not part of the answer. diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts index 682a5f0dd2..643b620c0f 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts @@ -31,11 +31,11 @@ type CountTokens = ( ) => Promise; const candidate = ( - callMessagesCountTokens: CountTokens, + callAnthropicMessagesCountTokens: CountTokens, overrides: { upstreamId?: string; endpoints?: ModelEndpoints } = {}, ): ModelCandidate => { const upstreamId = overrides.upstreamId ?? 'up_a'; - const endpoints = overrides.endpoints ?? { messages: {} }; + const endpoints = overrides.endpoints ?? { anthropicMessages: {} }; return { provider: { upstreamId, kind: 'claude-code', name: upstreamId, @@ -43,7 +43,7 @@ const candidate = ( // rather than proving only that the allowlist did. inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callMessagesCountTokens: callMessagesCountTokens as never }), + instance: stubProvider({ callAnthropicMessagesCountTokens: callAnthropicMessagesCountTokens as never }), }, model: stubInternalModel( { @@ -78,7 +78,7 @@ const count = async ( geminiGenerateContentCountTokensPipeline(request), move({ 'ingress.http.headers': headers, - 'ingress.chat.sourceProtocol': 'gemini', + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', 'request.chat.geminiGenerateContent': request, 'serve.model': 'gemini-model', }) as never, @@ -205,7 +205,7 @@ describe('the gemini count-tokens chain', () => { // Only an upstream's own Messages endpoint measures, so a candidate that would serve // generation over a translated wire cannot serve this. it('refuses a candidate no wire can measure on, naming the action', async () => { - resolves([candidate(async () => answered({ input_tokens: 0 }), { endpoints: { chatCompletions: {} } })]); + resolves([candidate(async () => answered({ input_tokens: 0 }), { endpoints: { openaiChatCompletions: {} } })]); const { facts } = await count(); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index a979f0f85a..bbd5b4bbe6 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -13,7 +13,7 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; -import { GEMINI_MISSING_TERMINAL_MESSAGE, type GeminiGenerateContentPayload, type GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; +import { GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, type GeminiGenerateContentPayload, type GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; import { directFetcher, type ModelCandidate, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -38,20 +38,20 @@ const resolveAttempt = (selector: { readonly upstreamId: string }): ModelCandida return found; }; -type CallChatCompletions = ( +type CallOpenAIChatCompletions = ( model: unknown, body: unknown, signal: AbortSignal | undefined, opts: UpstreamCallOptions, ) => Promise>; -const candidate = (upstream: string, callChatCompletions: CallChatCompletions): ModelCandidate => { - const endpoints = { chatCompletions: {} }; +const candidate = (upstream: string, callOpenAIChatCompletions: CallOpenAIChatCompletions): ModelCandidate => { + const endpoints = { openaiChatCompletions: {} }; return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callChatCompletions: callChatCompletions as never }), + instance: stubProvider({ callOpenAIChatCompletions: callOpenAIChatCompletions as never }), }, model: stubInternalModel({ id: 'gemini-2.5-pro', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'gemini-2.5-pro', endpoints }) } }, upstream), fetcher: directFetcher, @@ -151,7 +151,7 @@ const serveWith = async (gateway: ReturnType, facts: const entryFacts = (overrides: Record = {}) => ({ 'ingress.chat.geminiGenerateContent.wantsStream': true, - 'ingress.chat.sourceProtocol': 'gemini', + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', 'ingress.http.headers': [], 'request.chat.geminiGenerateContent': payload, 'serve.model': 'gemini-2.5-pro', @@ -401,7 +401,7 @@ describe('the gemini pipeline', () => { const { facts, drain } = await serve(entryFacts()); - await expect(collect(facts['response.chat.geminiGenerateContent.rendered'])).rejects.toThrow(GEMINI_MISSING_TERMINAL_MESSAGE); + await expect(collect(facts['response.chat.geminiGenerateContent.rendered'])).rejects.toThrow(GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE); await drain(); }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts index b22fecea8e..f19bbc5424 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts @@ -31,7 +31,7 @@ type CountTokens = ( ) => Promise; const candidate = ( - callMessagesCountTokens: CountTokens, + callAnthropicMessagesCountTokens: CountTokens, overrides: { upstreamId?: string; endpoints?: ModelEndpoints; @@ -40,7 +40,7 @@ const candidate = ( } = {}, ): ModelCandidate => { const upstreamId = overrides.upstreamId ?? 'up_a'; - const endpoints = overrides.endpoints ?? { messages: {} }; + const endpoints = overrides.endpoints ?? { anthropicMessages: {} }; const enabledFlags = overrides.enabledFlags ?? new Set(); return { provider: { @@ -49,7 +49,7 @@ const candidate = ( // rather than proving only that the allowlist did. inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callMessagesCountTokens: callMessagesCountTokens as never }), + instance: stubProvider({ callAnthropicMessagesCountTokens: callAnthropicMessagesCountTokens as never }), }, model: stubInternalModel( { @@ -88,7 +88,7 @@ const count = async ( anthropicMessagesCountTokensPipeline(request), move({ 'ingress.http.headers': headers, - 'ingress.chat.sourceProtocol': 'messages', + 'ingress.chat.sourceProtocol': 'anthropicMessages', 'request.chat.anthropicMessages': request, 'serve.model': request.model, }) as never, @@ -314,7 +314,7 @@ describe('the messages count-tokens chain', () => { // candidate that would serve generation over a translated wire cannot serve this, and the // refusal names the endpoint the client actually addressed. it('refuses a candidate no wire can measure on, naming the count endpoint', async () => { - resolves([candidate(async () => counted(0), { endpoints: { chatCompletions: {} } })]); + resolves([candidate(async () => counted(0), { endpoints: { openaiChatCompletions: {} } })]); const { facts } = await count(); diff --git a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts index e5276912e0..c8a524d75b 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts @@ -14,7 +14,7 @@ import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { directFetcher, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -25,15 +25,15 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal let live: readonly ModelCandidate[] = []; -type CallMessages = ( +type CallAnthropicMessages = ( model: unknown, body: unknown, signal: AbortSignal | undefined, opts: AnthropicMessagesUpstreamCallOptions, ) => Promise>; -const candidate = (callMessages: CallMessages, upstreamId = 'up_a'): ModelCandidate => { - const endpoints = { messages: {} }; +const candidate = (callAnthropicMessages: CallAnthropicMessages, upstreamId = 'up_a'): ModelCandidate => { + const endpoints = { anthropicMessages: {} }; return { provider: { upstreamId, kind: 'claude-code', name: upstreamId, @@ -41,7 +41,7 @@ const candidate = (callMessages: CallMessages, upstreamId = 'up_a'): ModelCandid // rather than proving only that the allowlist did. inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callMessages: callMessages as never }), + instance: stubProvider({ callAnthropicMessages: callAnthropicMessages as never }), }, model: stubInternalModel( { id: 'claude-model', endpoints, providerModels: { [upstreamId]: stubProviderModel({ id: 'claude-model', endpoints }) } }, @@ -111,7 +111,7 @@ const serveWith = async ( anthropicMessagesServePipeline(payload), move({ 'ingress.http.headers': headers, - 'ingress.chat.sourceProtocol': 'messages', + 'ingress.chat.sourceProtocol': 'anthropicMessages', 'ingress.chat.anthropicMessages.wantsStream': wantsStream, 'request.chat.anthropicMessages': payload, 'serve.model': 'claude-model', @@ -391,6 +391,6 @@ describe('the messages chain', () => { const { facts } = await serve(true); - await expect(collect(facts['response.chat.anthropicMessages.rendered'])).rejects.toThrow(MESSAGES_MISSING_TERMINAL_MESSAGE); + await expect(collect(facts['response.chat.anthropicMessages.rendered'])).rejects.toThrow(ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE); }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts index dee235c746..42cb1f1d48 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts @@ -13,14 +13,14 @@ import { SUMMARY_PREFIX, } from '../../../../src/data-plane/chat/openai-responses/compact-shim.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../src/shared/base64url-json.ts'; -import type { CanonicalResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; import { assertEquals } from '@floway-dev/test-utils'; -const turnFor = (payload: Partial = {}): CanonicalResponsesPayload => - summarizationTurnFor({ model: 'test-model', input: [], ...payload } as CanonicalResponsesPayload); +const turnFor = (payload: Partial = {}): CanonicalOpenAIResponsesPayload => + summarizationTurnFor({ model: 'test-model', input: [], ...payload } as CanonicalOpenAIResponsesPayload); /** The items a turn was rewritten into, as the assertions below read them. */ -const itemsOf = (payload: CanonicalResponsesPayload): { type: string; role?: string; content?: { type: string; text: string }[] }[] => +const itemsOf = (payload: CanonicalOpenAIResponsesPayload): { type: string; role?: string; content?: { type: string; text: string }[] }[] => payload.input as { type: string; role?: string; content?: { type: string; text: string }[] }[]; /** What the summarization turn produced, as an upstream states it. */ diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts index 2c23593a76..f45b328a14 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts @@ -544,9 +544,9 @@ test('POST /v1/responses with an unresolvable previous_response_id renders the v test('POST /v1/responses replays a stored turn when the next one names it as previous_response_id', async () => { installRepo(); - const observedBodies: Array> = []; - const callResponses = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); + const observedBodies: Array> = []; + const callOpenAIResponses = vi.fn(async (_model, body): Promise => { + observedBodies.push(body as Omit); return { action: 'generate', ok: true, events: makeProviderEvents(completedEvents(`resp_upstream_${observedBodies.length}`)), @@ -554,7 +554,7 @@ test('POST /v1/responses replays a stored turn when the next one names it as pre headers: new Headers(), }; }); - const candidate = makeCandidate({ callResponses }); + const candidate = makeCandidate({ callOpenAIResponses }); queueResolution([candidate]); queueResolution([candidate]); const app = makeApp(); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/run-dump_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/run-dump_test.ts index 9d9990866c..4586756deb 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/run-dump_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/run-dump_test.ts @@ -7,10 +7,10 @@ import { test, vi } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; import { eventsOf, installDumpStubs, runRecordOf } from '../../../dump/test-fixtures.ts'; -import { copilotModels, flushAsyncWork, requestApp, setupAppTest, sseResponsesResponse } from '../../../test-utils/app.ts'; +import { copilotModels, flushAsyncWork, requestApp, setupAppTest, sseOpenAIResponsesResponse } from '../../../test-utils/app.ts'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; -const withResponsesUpstream = async (run: () => Promise): Promise => +const withOpenAIResponsesUpstream = async (run: () => Promise): Promise => await withMockedFetch( async request => { const url = new URL(request.url); @@ -22,7 +22,7 @@ const withResponsesUpstream = async (run: () => Promise): Promise => return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); } if (url.pathname === '/responses') { - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: 'resp_run_dump', object: 'response', model: 'gpt-direct-responses', @@ -42,7 +42,7 @@ test('POST /v1/responses records the run it was, not an empty record', async () await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); - await withResponsesUpstream(async () => { + await withOpenAIResponsesUpstream(async () => { const response = await requestApp('/v1/responses', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index 52a40cd22e..de6889d046 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -2,14 +2,14 @@ import type { ExecutionContext } from 'hono'; import { onTestFinished, test, vi } from 'vitest'; import { app } from '../../../../src/app.ts'; -import { hashResponsesItem } from '../../../../src/data-plane/chat/openai-responses/items/identity.ts'; +import { hashOpenAIResponsesItem } from '../../../../src/data-plane/chat/openai-responses/items/identity.ts'; import { KEEP_ALIVE_EVENT_TYPE } from '../../../../src/data-plane/chat/openai-responses/websocket.ts'; import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../../../src/data-plane/shared/sse.ts'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../../src/repo/usage-metrics.ts'; import { eventsOf, installDumpStubs, runRecordOf } from '../../../dump/test-fixtures.ts'; import { FakeTime } from '../../../test-time.ts'; -import { buildCodexUpstreamRecord, codexModels, copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseResponsesResponse } from '../../../test-utils/app.ts'; +import { buildCodexUpstreamRecord, codexModels, copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseOpenAIResponsesResponse } from '../../../test-utils/app.ts'; import { installWorkerWebSocketRuntime, type TestWorkerWebSocket } from '../../../test-utils/worker-websocket.ts'; import { assert, assertEquals, assertExists, assertStringIncludes, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -65,7 +65,7 @@ const terminalResponseId = (messages: readonly Record[]): strin return id; }; -const connectResponsesWebSocket = async (apiKey: string, upgradeHeaders: Record = {}): Promise => { +const connectOpenAIResponsesWebSocket = async (apiKey: string, upgradeHeaders: Record = {}): Promise => { const executionCtx = { waitUntil: () => {}, passThroughOnException: () => {}, @@ -105,7 +105,7 @@ const withWorkerWebSocketRuntime = async (run: () => Promise): Promise } }; -const withSuccessfulResponsesUpstream = async (run: () => Promise): Promise => +const withSuccessfulOpenAIResponsesUpstream = async (run: () => Promise): Promise => await withMockedFetch( async request => { const url = new URL(request.url); @@ -117,7 +117,7 @@ const withSuccessfulResponsesUpstream = async (run: () => Promise): Promis return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); } if (url.pathname === '/responses') { - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: 'resp_ws_policy_refresh', object: 'response', model: 'gpt-direct-responses', @@ -132,7 +132,7 @@ const withSuccessfulResponsesUpstream = async (run: () => Promise): Promis run, ); -const completeResponsesTurn = async ( +const completeOpenAIResponsesTurn = async ( client: TestWorkerWebSocket, eventId: string, ): Promise => { @@ -162,7 +162,7 @@ test('Responses WebSocket forwards stream events, echoes event_id, and ends the return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); } if (url.pathname === '/responses') { - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: 'resp_ws', object: 'response', model: 'gpt-direct-responses', @@ -175,7 +175,7 @@ test('Responses WebSocket forwards stream events, echoes event_id, and ends the throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const raw = recordRawMessages(client); const received = waitForMessages(client, messages => messages.some(isTerminalResponseEvent)); @@ -215,12 +215,12 @@ test('Responses WebSocket starts capturing on the next turn when dump retention const { apiKey, repo } = await setupAppTest(); const dumps = installDumpStubs(initDumpStore, initDumpBroker); - await withSuccessfulResponsesUpstream( + await withSuccessfulOpenAIResponsesUpstream( async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); - await completeResponsesTurn(client, 'capture-after-enable'); + await completeOpenAIResponsesTurn(client, 'capture-after-enable'); await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); const stored = dumps.stored[0]; @@ -256,14 +256,14 @@ test('Responses WebSocket stops capturing on the next turn when dump retention i await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); - await withSuccessfulResponsesUpstream( + await withSuccessfulOpenAIResponsesUpstream( async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); - await completeResponsesTurn(client, 'captured-before-disable'); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); + await completeOpenAIResponsesTurn(client, 'captured-before-disable'); await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: null }); - await completeResponsesTurn(client, 'not-captured-after-disable'); + await completeOpenAIResponsesTurn(client, 'not-captured-after-disable'); assertEquals(dumps.stored.length, 1); client.close(); @@ -276,12 +276,12 @@ test('Responses WebSocket dump responseBytes equals the UTF-8 payload bytes sent await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); - await withSuccessfulResponsesUpstream( + await withSuccessfulOpenAIResponsesUpstream( async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const recorded = recordRawMessages(client); try { - await completeResponsesTurn(client, '响应-byte-count'); + await completeOpenAIResponsesTurn(client, '响应-byte-count'); await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); const expectedBytes = recorded.messages.reduce( @@ -316,12 +316,12 @@ test('Responses WebSocket records one frame per event it sent', async () => { await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); - await withSuccessfulResponsesUpstream( + await withSuccessfulOpenAIResponsesUpstream( async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const recorded = recordRawMessages(client); try { - await completeResponsesTurn(client, 'one-frame-per-event'); + await completeOpenAIResponsesTurn(client, 'one-frame-per-event'); await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); // A keep-alive is the transport's own idle write rather than a frame of the turn, so @@ -349,10 +349,10 @@ test('Responses WebSocket settles a streamed turn once, with what the chain read const dumps = installDumpStubs(initDumpStore, initDumpBroker); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); - await withSuccessfulResponsesUpstream( + await withSuccessfulOpenAIResponsesUpstream( async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); - await completeResponsesTurn(client, 'settled-turn'); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); + await completeOpenAIResponsesTurn(client, 'settled-turn'); client.close(); }), ); @@ -380,9 +380,9 @@ test('Responses WebSocket settles a streamed turn once, with what the chain read test('Responses WebSocket rejects the next turn after its API key is rotated', async () => { const { apiKey, repo } = await setupAppTest(); - await withSuccessfulResponsesUpstream( + await withSuccessfulOpenAIResponsesUpstream( async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); await repo.apiKeys.save({ ...apiKey, key: 'rotated-api-key' }); const received = waitForMessages(client, messages => messages.length === 1); @@ -424,7 +424,7 @@ test('Responses WebSocket reports a failed turn when an output item cannot be pe return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); } if (url.pathname === '/responses') { - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: 'resp_ws_persist_failure', object: 'response', model: 'gpt-direct-responses', @@ -443,7 +443,7 @@ test('Responses WebSocket reports a failed turn when an output item cannot be pe throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const received = waitForMessages(client, messages => messages.some(message => message.type === 'error')); client.send(JSON.stringify({ @@ -547,7 +547,7 @@ test('Responses WebSocket keep-alive waits for the first event and takes a slot throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const messages: Record[] = []; const onMessage = (event: Event): void => { messages.push(JSON.parse((event as MessageEvent).data) as Record); @@ -620,7 +620,7 @@ test('Responses WebSocket keep-alive waits for the first event and takes a slot test('Responses WebSocket returns OpenAI-style error envelopes for unsupported client events', async () => { const { apiKey } = await setupAppTest(); await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const received = waitForMessages(client, messages => messages.length === 1); client.send(JSON.stringify({ type: 'session.update', event_id: 'evt_bad' })); @@ -641,7 +641,7 @@ test('Responses WebSocket returns OpenAI-style error envelopes for unsupported c test('Responses WebSocket returns invalid_request_error for malformed client messages', async () => { const { apiKey } = await setupAppTest(); await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const invalidJson = waitForMessages(client, messages => messages.length === 1); client.send('{bad json'); @@ -732,7 +732,7 @@ test('Responses WebSocket forwards HTTP failures with status, error.code, and ev throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const received = waitForMessages(client, messages => messages.length === 1); client.send(JSON.stringify({ @@ -774,7 +774,7 @@ test('Responses WebSocket dump responseBytes counts an error envelope sent downs throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const recorded = recordRawMessages(client); try { const received = waitForMessages(client, messages => messages.length === 1); @@ -819,7 +819,7 @@ test('Responses WebSocket store:false keeps session snapshots without durable re if (url.pathname === '/responses') { upstreamBodies.push(JSON.parse(await request.text())); const turn = upstreamBodies.length; - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: `resp_ws_store_false_${turn}`, object: 'response', model: 'gpt-direct-responses', @@ -837,7 +837,7 @@ test('Responses WebSocket store:false keeps session snapshots without durable re throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const firstTerminal = waitForMessages(client, messages => messages.some(isTerminalResponseEvent)); client.send(JSON.stringify({ type: 'response.create', @@ -859,7 +859,7 @@ test('Responses WebSocket store:false keeps session snapshots without durable re assert(firstOutput.item.id !== 'assistant_ws_store_false_1', 'expected Copilot to replace the raw message id'); assertEquals(await repo.openaiResponsesItems.lookupMany(apiKey.id, [firstOutput.item.id], 0), []); assertEquals( - await repo.openaiResponsesItems.lookupManyByItemHash(apiKey.id, [await hashResponsesItem({ type: 'message', role: 'user', content: 'first question' })], 0), + await repo.openaiResponsesItems.lookupManyByItemHash(apiKey.id, [await hashOpenAIResponsesItem({ type: 'message', role: 'user', content: 'first question' })], 0), [], ); @@ -886,7 +886,7 @@ test('Responses WebSocket store:false keeps session snapshots without durable re ['message', 'user', 'follow-up'], ]); - const sessionB = await connectResponsesWebSocket(apiKey.key); + const sessionB = await connectOpenAIResponsesWebSocket(apiKey.key); const missingError = waitForMessages(sessionB, messages => messages.length === 1); sessionB.send(JSON.stringify({ type: 'response.create', @@ -938,7 +938,7 @@ test('Responses WebSocket evicts a failed continuation target so the next attemp headers: { 'content-type': 'Application/Problem+JSON; charset=utf-8' }, }); } - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: `resp_ws_evict_${responseCalls}`, object: 'response', model: 'gpt-direct-responses', @@ -956,7 +956,7 @@ test('Responses WebSocket evicts a failed continuation target so the next attemp throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const firstTerminal = waitForMessages(client, messages => messages.some(isTerminalResponseEvent)); client.send(JSON.stringify({ type: 'response.create', @@ -1051,7 +1051,7 @@ test('Responses WebSocket evicts a continuation that failed through a streamed t { data: '[DONE]' }, ]); } - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: `resp_ws_evict_streamed_${responseCalls}`, object: 'response', model: 'gpt-direct-responses', @@ -1069,7 +1069,7 @@ test('Responses WebSocket evicts a continuation that failed through a streamed t throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const firstTerminal = waitForMessages(client, messages => messages.some(isTerminalResponseEvent)); client.send(JSON.stringify({ type: 'response.create', @@ -1137,7 +1137,7 @@ test('Responses WebSocket store:true durable snapshots can chain through local s } if (url.pathname === '/responses') { turn += 1; - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: `resp_ws_durable_${turn}`, object: 'response', model: 'gpt-direct-responses', @@ -1155,7 +1155,7 @@ test('Responses WebSocket store:true durable snapshots can chain through local s throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const firstTerminal = waitForMessages(client, messages => messages.some(isTerminalResponseEvent)); client.send(JSON.stringify({ type: 'response.create', response: { model: 'gpt-direct-responses', input: 'first' } })); const firstMessages = await firstTerminal; @@ -1230,7 +1230,7 @@ test('Responses WebSocket makes a done reasoning item reusable from a fresh conn }), { headers: { 'content-type': 'text/event-stream' } }); } resolveSecondBody(body); - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: 'resp_second', object: 'response', model: 'gpt-direct-responses', @@ -1245,7 +1245,7 @@ test('Responses WebSocket makes a done reasoning item reusable from a fresh conn }, async () => await withWorkerWebSocketRuntime(async () => { try { - firstClient = await connectResponsesWebSocket(apiKey.key); + firstClient = await connectOpenAIResponsesWebSocket(apiKey.key); const firstItemDone = waitForMessages(firstClient, messages => messages.some(message => message.type === 'response.output_item.done')); firstClient.send(JSON.stringify({ @@ -1259,7 +1259,7 @@ test('Responses WebSocket makes a done reasoning item reusable from a fresh conn assert(done.item.encrypted_content !== originalReasoning.encrypted_content); firstClient.close(); - secondClient = await connectResponsesWebSocket(apiKey.key); + secondClient = await connectOpenAIResponsesWebSocket(apiKey.key); secondClient.send(JSON.stringify({ type: 'response.create', response: { @@ -1281,8 +1281,8 @@ test('Responses WebSocket makes a done reasoning item reusable from a fresh conn ); }); -// Exercises the session-level item cache directly: createResponsesWsSession -// builds a per-session MemoryStatefulResponsesBacking that mirrors every +// Exercises the session-level item cache directly: createOpenAIResponsesWsSession +// builds a per-session MemoryStatefulOpenAIResponsesBacking that mirrors every // durable write. Wiping the D1-backed repo between turns proves the second // message resolves the prior snapshot purely from in-RAM session cache. // A fresh WS session after the repo wipe MUST NOT see it (the cache is @@ -1304,7 +1304,7 @@ test('Responses WebSocket session-level store: second message resolves prior ite if (url.pathname === '/responses') { upstreamBodies.push(JSON.parse(await request.text())); const turn = upstreamBodies.length; - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: `resp_session_${turn}`, object: 'response', model: 'gpt-direct-responses', @@ -1322,7 +1322,7 @@ test('Responses WebSocket session-level store: second message resolves prior ite throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const sessionA = await connectResponsesWebSocket(apiKey.key); + const sessionA = await connectOpenAIResponsesWebSocket(apiKey.key); const firstTerminal = waitForMessages(sessionA, messages => messages.some(isTerminalResponseEvent)); sessionA.send(JSON.stringify({ type: 'response.create', @@ -1371,7 +1371,7 @@ test('Responses WebSocket session-level store: second message resolves prior ite // A fresh WS session for the same api key has its own empty cache; with // the repo wiped, the snapshot is unreachable. - const sessionB = await connectResponsesWebSocket(apiKey.key); + const sessionB = await connectOpenAIResponsesWebSocket(apiKey.key); const missingError = waitForMessages(sessionB, messages => messages.length === 1); sessionB.send(JSON.stringify({ type: 'response.create', @@ -1400,9 +1400,9 @@ test('Responses WebSocket session-level store: second message resolves prior ite test('Responses WebSocket aborts the in-flight Responses request when the client closes', async () => { const { apiKey } = await setupAppTest(); - let resolveResponsesStarted: (() => void) | undefined; + let resolveOpenAIResponsesStarted: (() => void) | undefined; const openaiResponsesStarted = new Promise(resolve => { - resolveResponsesStarted = resolve; + resolveOpenAIResponsesStarted = resolve; }); let resolveUpstreamAborted: (() => void) | undefined; const upstreamAborted = new Promise(resolve => { @@ -1420,11 +1420,11 @@ test('Responses WebSocket aborts the in-flight Responses request when the client return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); } if (url.pathname === '/responses') { - resolveResponsesStarted?.(); + resolveOpenAIResponsesStarted?.(); return await new Promise(resolve => { request.signal.addEventListener('abort', () => { resolveUpstreamAborted?.(); - resolve(sseResponsesResponse({ + resolve(sseOpenAIResponsesResponse({ id: 'resp_ws_abort', object: 'response', model: 'gpt-direct-responses', @@ -1438,7 +1438,7 @@ test('Responses WebSocket aborts the in-flight Responses request when the client throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); client.send(JSON.stringify({ type: 'response.create', response: { @@ -1459,7 +1459,7 @@ test('Responses WebSocket aborts the in-flight Responses request when the client // alongside its sendError / dump.failed / dump.finalize it calls // `recordFailedRequest(ctx, ctx.attempt.telemetry)`, and the row lands attributed // to the candidate that was being dialled when it threw. The throw here is a real -// one: a refusal whose body cannot be read. `callResponsesUpstream` stamps this +// one: a refusal whose body cannot be read. `callOpenAIResponsesUpstream` stamps this // candidate's telemetry before it dials and then reads the refusal's own words, so // the read's rejection is a mid-attempt throw with attribution already in place. test('Responses WebSocket outer catch records a failed perf sample attributed to the throwing candidate', async () => { @@ -1486,7 +1486,7 @@ test('Responses WebSocket outer catch records a failed perf sample attributed to throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectResponsesWebSocket(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); const received = waitForMessages(client, messages => messages.length === 1); client.send(JSON.stringify({ type: 'response.create', @@ -1547,7 +1547,7 @@ test('Responses WebSocket dispatches each Codex turn with the metadata blob that if (url.pathname === '/backend-api/codex/responses') { upstreamBodies.push(JSON.parse(await request.text()) as Record); const turn = upstreamBodies.length; - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: `resp_codex_ws_${turn}`, object: 'response', model: 'gpt-5.4', @@ -1565,7 +1565,7 @@ test('Responses WebSocket dispatches each Codex turn with the metadata blob that throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const socket = await connectResponsesWebSocket(apiKey.key, { + const socket = await connectOpenAIResponsesWebSocket(apiKey.key, { 'session-id': 'codex-session', 'thread-id': 'codex-thread', 'x-codex-window-id': 'codex-thread:0', diff --git a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts index a8aebaff51..260b9ef393 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts @@ -18,8 +18,8 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { CanonicalResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -31,13 +31,13 @@ let live: readonly ModelCandidate[] = []; const candidate = ( calls: { - callResponses?: (model: unknown, body: unknown, action: unknown) => Promise; - callMessages?: (model: unknown, body: unknown) => Promise>; + callOpenAIResponses?: (model: unknown, body: unknown, action: unknown) => Promise; + callAnthropicMessages?: (model: unknown, body: unknown) => Promise>; }, overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, ): ModelCandidate => { const upstreamId = overrides.upstreamId ?? 'up_a'; - const endpoints = overrides.endpoints ?? { responses: {} }; + const endpoints = overrides.endpoints ?? { openaiResponses: {} }; const enabledFlags = overrides.enabledFlags ?? new Set(); return { provider: { @@ -78,7 +78,7 @@ const compaction = (overrides: Partial = {}): O const compacts = ( seen: { body?: Record; action?: unknown }, overrides: Partial = {}, -) => async (_model: unknown, body: unknown, action: unknown): Promise => { +) => async (_model: unknown, body: unknown, action: unknown): Promise => { seen.body = body as Record; seen.action = action; return { action: 'compact', ok: true, result: compaction(overrides), modelKey: 'responses-model-key' }; @@ -86,7 +86,7 @@ const compacts = ( /** A generate turn over the Responses wire that produces one assistant message. */ const generates = (text: string, seen: { body?: Record; action?: unknown } = {}) => - async (_model: unknown, body: unknown, action: unknown): Promise => { + async (_model: unknown, body: unknown, action: unknown): Promise => { seen.body = body as Record; seen.action = action; const response: OpenAIResponsesResult = { @@ -143,19 +143,19 @@ const payload = { model: 'responses-model', input: [{ type: 'message', role: 'user', content: 'a long conversation' }], store: false, -} as unknown as CanonicalResponsesPayload; +} as unknown as CanonicalOpenAIResponsesPayload; const settlements: { billable: unknown; failed: boolean }[] = []; let gateway = mockChatGatewayCtx({ wantsStream: false }); -const compact = async (request: CanonicalResponsesPayload = payload) => { +const compact = async (request: CanonicalOpenAIResponsesPayload = payload) => { gateway = mockChatGatewayCtx({ wantsStream: false }); const outcome = await run( openaiResponsesCompactPipeline(request), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], - 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.sourceProtocol': 'openaiResponses', 'request.chat.openaiResponses': request, 'serve.model': request.model, }) as never, @@ -210,7 +210,7 @@ describe('the responses compaction chain', () => { // with one this gateway minted and completes the resource this endpoint answers with. it('dials a compaction, and answers the compaction resource under its own id', async () => { const seen: { body?: Record; action?: unknown } = {}; - resolves([candidate({ callResponses: compacts(seen) })]); + resolves([candidate({ callOpenAIResponses: compacts(seen) })]); const { facts } = await compact(); @@ -230,9 +230,9 @@ describe('the responses compaction chain', () => { // it rejects, and `stream` describes a delivery this operation does not make. it('sends neither store nor stream on the compaction wire', async () => { const seen: { body?: Record } = {}; - resolves([candidate({ callResponses: compacts(seen) })]); + resolves([candidate({ callOpenAIResponses: compacts(seen) })]); - await compact({ ...payload, stream: false } as unknown as CanonicalResponsesPayload); + await compact({ ...payload, stream: false } as unknown as CanonicalOpenAIResponsesPayload); expect(seen.body).not.toHaveProperty('store'); expect(seen.body).not.toHaveProperty('stream'); @@ -244,7 +244,7 @@ describe('the responses compaction chain', () => { // wire generation would have used — which is the whole of what the action pivot was. it('simulates a compaction over a candidate with no compaction wire', async () => { const seen: { body?: Record } = {}; - resolves([candidate({ callMessages: anthropicMessagesTurn('CONDENSED SUMMARY', seen) }, { endpoints: { messages: {} } })]); + resolves([candidate({ callAnthropicMessages: anthropicMessagesTurn('CONDENSED SUMMARY', seen) }, { endpoints: { anthropicMessages: {} } })]); const { facts } = await compact({ ...payload, @@ -252,7 +252,7 @@ describe('the responses compaction chain', () => { { type: 'message', role: 'user', content: 'real history' }, { type: 'compaction_trigger' }, ], - } as unknown as CanonicalResponsesPayload); + } as unknown as CanonicalOpenAIResponsesPayload); // The compactor's prompt reached the upstream, and the control item that asked for the // compaction did not. @@ -271,7 +271,7 @@ describe('the responses compaction chain', () => { it('simulates rather than dials when the operator asked it to', async () => { const seen: { body?: Record; action?: unknown } = {}; resolves([candidate( - { callResponses: generates('SIMULATED SUMMARY', seen) }, + { callOpenAIResponses: generates('SIMULATED SUMMARY', seen) }, { enabledFlags: new Set(['responses-compact-shim']) }, )]); @@ -293,14 +293,14 @@ describe('the responses compaction chain', () => { { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'THE EARLIER HISTORY' }] }, ]); resolves([candidate( - { callResponses: generates('S', seen) }, + { callOpenAIResponses: generates('S', seen) }, { enabledFlags: new Set(['responses-compact-shim']) }, )]); await compact({ ...payload, input: [{ type: 'compaction', id: 'cmp_prior', encrypted_content: encoded }], - } as unknown as CanonicalResponsesPayload); + } as unknown as CanonicalOpenAIResponsesPayload); expect(JSON.stringify(seen.body)).toContain('THE EARLIER HISTORY'); }); @@ -312,11 +312,11 @@ describe('the responses compaction chain', () => { const tried: string[] = []; resolves([ candidate( - { callResponses: async (...args) => { tried.push('mute'); return await generates('')(...args as [unknown, unknown, unknown]); } }, + { callOpenAIResponses: async (...args) => { tried.push('mute'); return await generates('')(...args as [unknown, unknown, unknown]); } }, { upstreamId: 'up_mute', enabledFlags: new Set(['responses-compact-shim']) }, ), candidate( - { callResponses: async (...args) => { tried.push('talkative'); return await generates('A SUMMARY')(...args as [unknown, unknown, unknown]); } }, + { callOpenAIResponses: async (...args) => { tried.push('talkative'); return await generates('A SUMMARY')(...args as [unknown, unknown, unknown]); } }, { upstreamId: 'up_talkative', enabledFlags: new Set(['responses-compact-shim']) }, ), ]); @@ -331,7 +331,7 @@ describe('the responses compaction chain', () => { it('fails a refused compaction over to the next candidate, and keeps the last refusal-s own status', async () => { resolves([ candidate({ - callResponses: async (): Promise => ({ + callOpenAIResponses: async (): Promise => ({ action: 'compact', ok: false, modelKey: 'k', response: new Response(JSON.stringify({ error: { message: 'slow down', type: 'rate_limit' } }), { status: 429, headers: { 'content-type': 'application/json' }, @@ -339,7 +339,7 @@ describe('the responses compaction chain', () => { }), }, { upstreamId: 'up_busy' }), candidate({ - callResponses: async (): Promise => ({ + callOpenAIResponses: async (): Promise => ({ action: 'compact', ok: false, modelKey: 'k', response: new Response(JSON.stringify({ error: { message: 'still no', type: 'rate_limit' } }), { status: 503, headers: { 'content-type': 'application/json' }, @@ -357,7 +357,7 @@ describe('the responses compaction chain', () => { // `Usage` has no `null` alternative on this resource and zeros are not its spelling, so an // upstream that stated no counts has left the gateway unable to answer. it('reports a compaction the upstream stated no counts for as the gateway-s own failure', async () => { - resolves([candidate({ callResponses: compacts({}, { usage: undefined }) })]); + resolves([candidate({ callOpenAIResponses: compacts({}, { usage: undefined }) })]); const { facts } = await compact(); @@ -371,7 +371,7 @@ describe('the responses compaction chain', () => { // the upstream actually ran — and a compaction that surfaced as failed belongs in the error // column rather than masquerading as a success. it('settles a compaction whose turn failed as a failure', async () => { - resolves([candidate({ callResponses: compacts({}, { status: 'failed' } as Partial) })]); + resolves([candidate({ callOpenAIResponses: compacts({}, { status: 'failed' } as Partial) })]); await compact(); @@ -379,7 +379,7 @@ describe('the responses compaction chain', () => { }); it('settles a compaction that ran as a success, billing what its envelope stated', async () => { - resolves([candidate({ callResponses: compacts({}) })]); + resolves([candidate({ callOpenAIResponses: compacts({}) })]); await compact(); @@ -396,7 +396,7 @@ describe('the responses compaction chain', () => { // this operation would answer 502 where the upstream had said nothing at all. it('hands the provider a body it can write into, children included', async () => { resolves([candidate({ - callResponses: async (model, body, action) => { + callOpenAIResponses: async (model, body, action) => { const item = (body as { input: Record[] }).input[0]!; item.copilot_cache_control = { type: 'ephemeral' }; return await compacts({})(model, body, action); diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index d291f52a3f..339f329f97 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -20,8 +20,8 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { ModelEndpoints, SseFrame } from '@floway-dev/protocols/common'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalResponsesPayload, type OpenAIResponsesCompactionResult, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; +import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesCompactionResult, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -33,13 +33,13 @@ let live: readonly ModelCandidate[] = []; const candidate = ( calls: { - callResponses?: (model: unknown, body: unknown, action: unknown) => Promise; - callMessages?: (model: unknown, body: unknown) => Promise>; + callOpenAIResponses?: (model: unknown, body: unknown, action: unknown) => Promise; + callAnthropicMessages?: (model: unknown, body: unknown) => Promise>; }, overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, ): ModelCandidate => { const upstreamId = overrides.upstreamId ?? 'up_a'; - const endpoints = overrides.endpoints ?? { responses: {} }; + const endpoints = overrides.endpoints ?? { openaiResponses: {} }; const enabledFlags = overrides.enabledFlags ?? new Set(); return { provider: { @@ -96,7 +96,7 @@ const completed = (text: string, usage?: OpenAIResponsesResult['usage']): OpenAI // The upstream's own stream, as the provider hands it up: typed frames, and the `done` frame // its SSE transport ended on. -const stream = (...events: readonly OpenAIResponsesStreamEvent[]): ProviderResponsesResult => ({ +const stream = (...events: readonly OpenAIResponsesStreamEvent[]): ProviderOpenAIResponsesResult => ({ action: 'generate', ok: true, modelKey: 'responses-model-key', @@ -138,28 +138,28 @@ const anthropicMessagesTurn = (text: string, seen: { body?: Record +const serve = async (wantsStream: boolean, request: CanonicalOpenAIResponsesPayload = payload) => await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream, request); const serveWith = async ( gateway: ReturnType, wantsStream: boolean, - request: CanonicalResponsesPayload = payload, + request: CanonicalOpenAIResponsesPayload = payload, ) => await run( openaiResponsesServePipeline(request), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], - 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.sourceProtocol': 'openaiResponses', 'ingress.chat.openaiResponses.wantsStream': wantsStream, 'request.chat.openaiResponses': request, 'serve.model': 'responses-model', @@ -218,7 +218,7 @@ const asksForCompaction = { { type: 'message', role: 'user', content: 'a long conversation' }, { type: 'compaction_trigger' }, ], -} as unknown as CanonicalResponsesPayload; +} as unknown as CanonicalOpenAIResponsesPayload; /** The compaction item out of what the client was answered with. */ const compactionItem = (answered: Record): { readonly encrypted_content: string } => { @@ -264,9 +264,9 @@ describe('the responses chain', () => { affinityPayload = { ...payload, input: [{ type: 'message', role: 'user', content: 'rewritten' }], - } as CanonicalResponsesPayload; + } as CanonicalOpenAIResponsesPayload; resolves([candidate({ - callResponses: async (_model, body, asks) => { + callOpenAIResponses: async (_model, body, asks) => { sent = body as Record; action = asks; return stream(completed('hi')); @@ -297,9 +297,9 @@ describe('the responses chain', () => { { type: 'message', role: 'assistant', content: [{ type: 'input_text', text: 'summary so far' }] }, { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'and then?' }] }, ], - } as unknown as CanonicalResponsesPayload; + } as unknown as CanonicalOpenAIResponsesPayload; resolves([candidate({ - callResponses: async (_model, body) => { + callOpenAIResponses: async (_model, body) => { sent = body as Record; return stream(completed('hi')); }, @@ -319,7 +319,7 @@ describe('the responses chain', () => { // Every event goes out under its own SSE name, and the client's stream ends on the literal // `[DONE]` the transport reads as the turn being over. it('writes the frames out when the client asked to stream', async () => { - resolves([candidate({ callResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); + resolves([candidate({ callOpenAIResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); const { facts } = await serve(true); const frames = withoutCarrier(await drain(facts['response.chat.openaiResponses.rendered'])); @@ -355,7 +355,7 @@ describe('the responses chain', () => { // this is what the client is given to carry back. A turn that handed back nothing would // leave the follow-up quoting it with no way to name the upstream that issued it. it('hands the turn-s own state back on an item of its own', async () => { - resolves([candidate({ callResponses: async () => stream(delta('hi'), completed('hi')) })]); + resolves([candidate({ callOpenAIResponses: async () => stream(delta('hi'), completed('hi')) })]); const gateway = mockChatGatewayCtx({ wantsStream: true }); const { facts } = await serveWith(gateway, true); @@ -379,7 +379,7 @@ describe('the responses chain', () => { // the reading is taken off the frames the client itself drove. it('bills what the terminal event reported, once the frames have run out', async () => { resolves([candidate({ - callResponses: async () => stream( + callOpenAIResponses: async () => stream( delta('hi'), completed('hi', { input_tokens: 11, output_tokens: 7, total_tokens: 18 }), ), @@ -401,7 +401,7 @@ describe('the responses chain', () => { // Time to first token is measured where the token is — a lifecycle envelope is not one, and // a run that never stamps it is recorded as having produced nothing to time. it('stamps the first frame that carried a generated token', async () => { - resolves([candidate({ callResponses: async () => stream(delta('hi'), completed('hi')) })]); + resolves([candidate({ callOpenAIResponses: async () => stream(delta('hi'), completed('hi')) })]); const gateway = mockChatGatewayCtx({ wantsStream: true }); const { facts } = await serveWith(gateway, true); @@ -415,7 +415,7 @@ describe('the responses chain', () => { // of the answer and the client is not shown it. Reading stops there, which also closes the // upstream rather than leaving the client's stream open behind a connection nobody reads. it('stops reading at the terminal event', async () => { - resolves([candidate({ callResponses: async () => stream(delta('hi'), completed('hi'), delta(' and more')) })]); + resolves([candidate({ callOpenAIResponses: async () => stream(delta('hi'), completed('hi'), delta(' and more')) })]); const { facts } = await serve(true); const frames = withoutCarrier(await drain(facts['response.chat.openaiResponses.rendered'])); @@ -434,7 +434,7 @@ describe('the responses chain', () => { // words — and the stream ends on that rather than on the terminator, because a stream that // ended on `[DONE]` is a stream that finished. it('tells a client already being streamed to that the stream never ended', async () => { - resolves([candidate({ callResponses: async () => stream(delta('hi')) })]); + resolves([candidate({ callOpenAIResponses: async () => stream(delta('hi')) })]); const { facts } = await serve(true); const frames = await drain(facts['response.chat.openaiResponses.rendered']); @@ -442,7 +442,7 @@ describe('the responses chain', () => { expect(frames.map(frame => frame.event)).toEqual(['response.output_text.delta', 'error']); expect(JSON.parse(frames[1]!.data)).toMatchObject({ type: 'error', - error: { message: RESPONSES_MISSING_TERMINAL_MESSAGE }, + error: { message: OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE }, }); // The client was answered, and the run still failed: what the frames said on the way out // is not what the row says the turn was. @@ -452,7 +452,7 @@ describe('the responses chain', () => { // The upstream speaks SSE whatever the client asked for, so a client that did not ask to // stream is answered from the same frames — folded here rather than read a second time. it('folds the frames into one response when the client did not', async () => { - resolves([candidate({ callResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); + resolves([candidate({ callOpenAIResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); const { facts } = await serve(false); @@ -480,7 +480,7 @@ describe('the responses chain', () => { usage: { input_tokens: 900, output_tokens: 40, total_tokens: 940 }, }; resolves([candidate({ - callResponses: async () => ({ + callOpenAIResponses: async () => ({ action: 'compact', ok: true, modelKey: 'responses-model-key', result: compaction, }), })]); @@ -498,7 +498,7 @@ describe('the responses chain', () => { // Content-length would misdescribe a body this gateway serialized itself; a vendor trace // is what a client and an operator both need to correlate a turn. it('forwards the upstream headers a client may see and drops the ones it may not', async () => { - resolves([candidate({ callResponses: async () => stream(completed('hi')) })]); + resolves([candidate({ callOpenAIResponses: async () => stream(completed('hi')) })]); const { facts } = await serve(true); @@ -508,7 +508,7 @@ describe('the responses chain', () => { it('answers an upstream refusal with its own status and words', async () => { resolves([candidate({ - callResponses: async () => ({ + callOpenAIResponses: async () => ({ action: 'generate', ok: false, modelKey: 'responses-model-key', @@ -529,8 +529,8 @@ describe('the responses chain', () => { it('fails a dial that never connected over to the next candidate', async () => { const tried: string[] = []; resolves([ - candidate({ callResponses: async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); } }, { upstreamId: 'up_dead' }), - candidate({ callResponses: async () => { tried.push('alive'); return stream(completed('hi')); } }, { upstreamId: 'up_alive' }), + candidate({ callOpenAIResponses: async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); } }, { upstreamId: 'up_dead' }), + candidate({ callOpenAIResponses: async () => { tried.push('alive'); return stream(completed('hi')); } }, { upstreamId: 'up_alive' }), ]); const { facts } = await serve(false); @@ -546,7 +546,7 @@ describe('the responses chain', () => { // answering the client with a 502 raised where nothing can explain it. it('hands the provider a body it can write into, children included', async () => { resolves([candidate({ - callResponses: async (_model, body) => { + callOpenAIResponses: async (_model, body) => { const item = (body as { input: Record[] }).input[0]!; item.copilot_cache_control = { type: 'ephemeral' }; return stream(completed('hi')); @@ -567,7 +567,7 @@ describe('the responses chain', () => { affinityPayload = asksForCompaction; resolves([candidate( { - callResponses: async (_model, body) => { + callOpenAIResponses: async (_model, body) => { sent = body as Record; return stream(completed('CONDENSED SUMMARY')); }, @@ -597,7 +597,7 @@ describe('the responses chain', () => { let sent: Record | undefined; resolves([candidate( { - callResponses: async (_model, body) => { + callOpenAIResponses: async (_model, body) => { sent = body as Record; return stream(completed('hello')); }, @@ -627,7 +627,7 @@ describe('the responses chain', () => { }; affinityPayload = asksForCompaction; resolves([candidate({ - callResponses: async (_model, body) => { + callOpenAIResponses: async (_model, body) => { sent = body as Record; return { action: 'compact', ok: true, modelKey: 'responses-model-key', result: upstreamCompaction }; }, @@ -647,8 +647,8 @@ describe('the responses chain', () => { const seen: { body?: Record } = {}; affinityPayload = asksForCompaction; resolves([candidate( - { callMessages: anthropicMessagesTurn('CONDENSED SUMMARY', seen) }, - { endpoints: { messages: {} } }, + { callAnthropicMessages: anthropicMessagesTurn('CONDENSED SUMMARY', seen) }, + { endpoints: { anthropicMessages: {} } }, )]); const gateway = mockChatGatewayCtx({ wantsStream: false }); @@ -678,11 +678,11 @@ describe('the responses chain', () => { }, { type: 'message', role: 'user', content: 'and then?' }, ], - } as unknown as CanonicalResponsesPayload; + } as unknown as CanonicalOpenAIResponsesPayload; affinityPayload = continues; resolves([candidate( { - callResponses: async (_model, body) => { + callOpenAIResponses: async (_model, body) => { sent = body as Record; return stream(completed('hi')); }, @@ -707,10 +707,10 @@ describe('the responses chain', () => { const continues = { ...payload, input: [{ type: 'compaction', id: 'cmp_prior', encrypted_content: encoded }], - } as unknown as CanonicalResponsesPayload; + } as unknown as CanonicalOpenAIResponsesPayload; affinityPayload = continues; resolves([candidate({ - callResponses: async (_model, body) => { + callOpenAIResponses: async (_model, body) => { sent = body as Record; return stream(completed('hi')); }, diff --git a/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts index 3c9b712edc..172aaecd80 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts @@ -11,7 +11,7 @@ import { expect, test } from 'vitest'; -import { copilotModels, requestApp, setupAppTest, sseResponsesResponse } from '../../test-utils/app.ts'; +import { copilotModels, requestApp, setupAppTest, sseOpenAIResponsesResponse } from '../../test-utils/app.ts'; import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; const ANSWER = 'hello from copilot'; @@ -52,7 +52,7 @@ test('a turn reaches Copilot over its own Responses endpoint and comes back as o (path, body) => { expect(path).toBe('/responses'); sent = body; - return sseResponsesResponse({ + return sseOpenAIResponsesResponse({ id: 'resp_route', object: 'response', model: 'gpt-responses', diff --git a/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts index 65f0825da9..6c794053e8 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts @@ -10,7 +10,7 @@ import { expect, test } from 'vitest'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { copilotModels, parseSSEText, requestApp, setupAppTest, sseChatCompletionsResponse, sseMessagesResponse } from '../../test-utils/app.ts'; +import { copilotModels, parseSSEText, requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseAnthropicMessagesResponse } from '../../test-utils/app.ts'; import { flushBackground } from '../../test-utils/background-tracker.ts'; import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -38,7 +38,7 @@ const withCopilot = async ( /** The answer a Chat Completions upstream gives, as the SSE every chat endpoint really speaks. */ const upstreamTurn = (model: string): Response => - sseChatCompletionsResponse({ + sseOpenAIChatCompletionsResponse({ id: 'chatcmpl_route', object: 'chat.completion', created: 1_772_000_000, @@ -190,7 +190,7 @@ test('a turn dialled over the Messages wire still answers the client in Chat Com (path, body) => { expect(path).toBe('/v1/messages'); expect(body.stream).toBe(true); - return sseMessagesResponse({ + return sseAnthropicMessagesResponse({ id: 'msg_route', type: 'message', role: 'assistant', diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts index ce88506579..53fba925b9 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts @@ -12,7 +12,7 @@ import { expect, test } from 'vitest'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { copilotModels, parseSSEText, requestApp, setupAppTest, sseChatCompletionsResponse, sseMessagesResponse } from '../../test-utils/app.ts'; +import { copilotModels, parseSSEText, requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseAnthropicMessagesResponse } from '../../test-utils/app.ts'; import { flushBackground } from '../../test-utils/background-tracker.ts'; import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -41,7 +41,7 @@ const withCopilot = async ( /** The answer a Chat Completions upstream gives, as the SSE every chat endpoint really speaks. * Gemini's first-preference wire, so this is what most of these turns are dialled on. */ const openaiChatCompletionsTurn = (): Response => - sseChatCompletionsResponse({ + sseOpenAIChatCompletionsResponse({ id: 'chatcmpl_route', object: 'chat.completion', created: 1_772_000_000, @@ -206,7 +206,7 @@ test('a turn dialled over the Messages wire still answers the client in Gemini', expect(path).toBe('/v1/messages'); expect(body.messages).toBeDefined(); expect(body.stream).toBe(true); - return sseMessagesResponse({ + return sseAnthropicMessagesResponse({ id: 'msg_route', type: 'message', role: 'assistant', diff --git a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts index 73c4f95ee7..24f9248717 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts @@ -10,7 +10,7 @@ import { expect, test } from 'vitest'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { copilotModels, parseSSEText, requestApp, setupAppTest, sseChatCompletionsResponse, sseMessagesResponse } from '../../test-utils/app.ts'; +import { copilotModels, parseSSEText, requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseAnthropicMessagesResponse } from '../../test-utils/app.ts'; import { flushBackground } from '../../test-utils/background-tracker.ts'; import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -38,7 +38,7 @@ const withCopilot = async ( /** The answer a Messages upstream gives, as the SSE every chat endpoint really speaks. */ const upstreamTurn = (model: string): Response => - sseMessagesResponse({ + sseAnthropicMessagesResponse({ id: 'msg_route', type: 'message', role: 'assistant', @@ -203,7 +203,7 @@ test('a turn dialled over the Chat Completions wire still answers the client in // The usage chunk is asked for by the wire the turn landed on, not by the chain it // arrived from — which is what keeps a translated turn metered at all. expect(body.stream_options).toEqual({ include_usage: true }); - return sseChatCompletionsResponse({ + return sseOpenAIChatCompletionsResponse({ id: 'chatcmpl_route', object: 'chat.completion', created: 1_772_000_000, diff --git a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts index 92b883468f..95ff9144eb 100644 --- a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts @@ -15,7 +15,7 @@ import { test, vi } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; -import { copilotModels, flushAsyncWork, requestApp, setupAppTest, sseChatCompletionsResponse, sseResponsesResponse } from '../../test-utils/app.ts'; +import { copilotModels, flushAsyncWork, requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseOpenAIResponsesResponse } from '../../test-utils/app.ts'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; /** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, @@ -102,7 +102,7 @@ const ANSWER = 'hello there'; const openaiChatCompletionsTurn = (stream: boolean) => async (apiKey: string): Promise => await withCopilot( [{ id: 'gpt-frames', supported_endpoints: ['/chat/completions'] }], - () => sseChatCompletionsResponse({ + () => sseOpenAIChatCompletionsResponse({ id: 'chatcmpl_frames', object: 'chat.completion', model: 'gpt-frames', @@ -119,7 +119,7 @@ const openaiChatCompletionsTurn = (stream: boolean) => async (apiKey: string): P const openaiResponsesTurn = (stream: boolean) => async (apiKey: string): Promise => await withCopilot( [{ id: 'gpt-responses-frames', supported_endpoints: ['/responses'] }], - () => sseResponsesResponse({ + () => sseOpenAIResponsesResponse({ id: 'resp_frames', object: 'response', model: 'gpt-responses-frames', diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 160080206a..3e434db4d7 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -27,8 +27,8 @@ import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsResult, OpenAIC import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocols/common'; import type { GeminiGenerateContentPayload, GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; import { PROMPT_TOO_LONG_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesResult, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { CanonicalResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { directFetcher, type FlagId, type ModelCandidate, type ProviderResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -46,9 +46,9 @@ const resolves = (candidates: readonly ModelCandidate[]): void => { }; interface Calls { - readonly callChatCompletions?: (model: unknown, body: unknown) => Promise>; - readonly callMessages?: (model: unknown, body: unknown) => Promise>; - readonly callResponses?: (model: unknown, body: unknown) => Promise; + readonly callOpenAIChatCompletions?: (model: unknown, body: unknown) => Promise>; + readonly callAnthropicMessages?: (model: unknown, body: unknown) => Promise>; + readonly callOpenAIResponses?: (model: unknown, body: unknown) => Promise; } /** A candidate reachable over exactly the endpoints it is given. Handing it one endpoint is @@ -129,7 +129,7 @@ const openaiResponsesResult = (text: string): OpenAIResponsesResult => ({ incomplete_details: null, }); -const openaiResponsesTurn = (text: string): ProviderResponsesResult => ({ +const openaiResponsesTurn = (text: string): ProviderOpenAIResponsesResult => ({ action: 'generate', ok: true, modelKey: 'k', @@ -185,41 +185,41 @@ const serve = async ( const openaiChatCompletionsPayload = { model: MODEL, messages: [{ role: 'user', content: 'hi' }] } as unknown as OpenAIChatCompletionsPayload; const anthropicMessagesPayload = { model: MODEL, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as AnthropicMessagesPayload; -const openaiResponsesPayload = { model: MODEL, input: [{ type: 'message', role: 'user', content: 'hi' }] } as unknown as CanonicalResponsesPayload; +const openaiResponsesPayload = { model: MODEL, input: [{ type: 'message', role: 'user', content: 'hi' }] } as unknown as CanonicalOpenAIResponsesPayload; const geminiGenerateContentPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiGenerateContentPayload; -const serveChatCompletions = async (payload: OpenAIChatCompletionsPayload = openaiChatCompletionsPayload) => +const serveOpenAIChatCompletions = async (payload: OpenAIChatCompletionsPayload = openaiChatCompletionsPayload) => await serve(openaiChatCompletionsServePipeline(payload), { 'ingress.http.headers': [], - 'ingress.chat.sourceProtocol': 'chatCompletions', + 'ingress.chat.sourceProtocol': 'openaiChatCompletions', 'ingress.chat.openaiChatCompletions.wantsStream': false, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false, 'request.chat.openaiChatCompletions': payload, 'serve.model': MODEL, }, payload); -const serveMessages = async (payload: AnthropicMessagesPayload = anthropicMessagesPayload) => +const serveAnthropicMessages = async (payload: AnthropicMessagesPayload = anthropicMessagesPayload) => await serve(anthropicMessagesServePipeline(payload), { 'ingress.http.headers': [], - 'ingress.chat.sourceProtocol': 'messages', + 'ingress.chat.sourceProtocol': 'anthropicMessages', 'ingress.chat.anthropicMessages.wantsStream': false, 'request.chat.anthropicMessages': payload, 'serve.model': MODEL, }, payload); -const serveResponses = async (payload: CanonicalResponsesPayload = openaiResponsesPayload) => +const serveOpenAIResponses = async (payload: CanonicalOpenAIResponsesPayload = openaiResponsesPayload) => await serve(openaiResponsesServePipeline(payload), { 'ingress.http.headers': [], - 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.sourceProtocol': 'openaiResponses', 'ingress.chat.openaiResponses.wantsStream': false, 'request.chat.openaiResponses': payload, 'serve.model': MODEL, }, payload); -const serveGemini = async (payload: GeminiGenerateContentPayload = geminiGenerateContentPayload) => +const serveGeminiGenerateContent = async (payload: GeminiGenerateContentPayload = geminiGenerateContentPayload) => await serve(geminiGenerateContentServePipeline(payload), { 'ingress.http.headers': [], - 'ingress.chat.sourceProtocol': 'gemini', + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', 'ingress.chat.geminiGenerateContent.wantsStream': false, 'request.chat.geminiGenerateContent': payload, 'serve.model': MODEL, @@ -256,102 +256,102 @@ beforeEach(() => { describe('a chat family reaching a candidate over another protocol', () => { it('serves /v1/chat/completions on a Messages-only candidate', async () => { - const callMessages = vi.fn(async () => anthropicMessagesTurn('hello')); - resolves([candidate('up_a', { messages: {} }, { callMessages })]); + const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); + resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); - const { facts, drain } = await serveChatCompletions(); + const { facts, drain } = await serveOpenAIChatCompletions(); - expect(callMessages).toHaveBeenCalledTimes(1); + expect(callAnthropicMessages).toHaveBeenCalledTimes(1); expect(facts['response.http.status']).toBe(200); expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/chat/completions on a Responses-only candidate', async () => { - const callResponses = vi.fn(async () => openaiResponsesTurn('hello')); - resolves([candidate('up_a', { responses: {} }, { callResponses })]); + const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); + resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); - const { facts, drain } = await serveChatCompletions(); + const { facts, drain } = await serveOpenAIChatCompletions(); - expect(callResponses).toHaveBeenCalledTimes(1); + expect(callOpenAIResponses).toHaveBeenCalledTimes(1); expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/messages on a Responses-only candidate', async () => { - const callResponses = vi.fn(async () => openaiResponsesTurn('hello')); - resolves([candidate('up_a', { responses: {} }, { callResponses })]); + const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); + resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); - const { facts, drain } = await serveMessages(); + const { facts, drain } = await serveAnthropicMessages(); - expect(callResponses).toHaveBeenCalledTimes(1); + expect(callOpenAIResponses).toHaveBeenCalledTimes(1); expect(anthropicMessagesText(facts['response.chat.anthropicMessages.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/messages on a Chat Completions-only candidate', async () => { - const callChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); - resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); + const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); - const { facts, drain } = await serveMessages(); + const { facts, drain } = await serveAnthropicMessages(); - expect(callChatCompletions).toHaveBeenCalledTimes(1); + expect(callOpenAIChatCompletions).toHaveBeenCalledTimes(1); expect(anthropicMessagesText(facts['response.chat.anthropicMessages.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/responses on a Messages-only candidate', async () => { - const callMessages = vi.fn(async () => anthropicMessagesTurn('hello')); - resolves([candidate('up_a', { messages: {} }, { callMessages })]); + const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); + resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); - const { facts, drain } = await serveResponses(); + const { facts, drain } = await serveOpenAIResponses(); - expect(callMessages).toHaveBeenCalledTimes(1); + expect(callAnthropicMessages).toHaveBeenCalledTimes(1); expect(openaiResponsesText(facts['response.chat.openaiResponses.rendered'])).toBe('hello'); await drain(); }); it('serves /v1/responses on a Chat Completions-only candidate', async () => { - const callChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); - resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); + const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); - const { facts, drain } = await serveResponses(); + const { facts, drain } = await serveOpenAIResponses(); - expect(callChatCompletions).toHaveBeenCalledTimes(1); + expect(callOpenAIChatCompletions).toHaveBeenCalledTimes(1); expect(openaiResponsesText(facts['response.chat.openaiResponses.rendered'])).toBe('hello'); await drain(); }); // Gemini has no wire of its own, so all three of its rows are translated ones. it('serves :generateContent on a Chat Completions-only candidate', async () => { - const callChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); - resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); + const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); - const { facts, drain } = await serveGemini(); + const { facts, drain } = await serveGeminiGenerateContent(); - expect(callChatCompletions).toHaveBeenCalledTimes(1); + expect(callOpenAIChatCompletions).toHaveBeenCalledTimes(1); expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); await drain(); }); it('serves :generateContent on a Messages-only candidate', async () => { - const callMessages = vi.fn(async () => anthropicMessagesTurn('hello')); - resolves([candidate('up_a', { messages: {} }, { callMessages })]); + const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); + resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); - const { facts, drain } = await serveGemini(); + const { facts, drain } = await serveGeminiGenerateContent(); - expect(callMessages).toHaveBeenCalledTimes(1); + expect(callAnthropicMessages).toHaveBeenCalledTimes(1); expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); await drain(); }); it('serves :generateContent on a Responses-only candidate', async () => { - const callResponses = vi.fn(async () => openaiResponsesTurn('hello')); - resolves([candidate('up_a', { responses: {} }, { callResponses })]); + const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); + resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); - const { facts, drain } = await serveGemini(); + const { facts, drain } = await serveGeminiGenerateContent(); - expect(callResponses).toHaveBeenCalledTimes(1); + expect(callOpenAIResponses).toHaveBeenCalledTimes(1); expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); await drain(); }); @@ -363,19 +363,19 @@ describe('the fork over wires', () => { // failover's own — the first candidate goes native and the second goes translated because // that is what each one's own endpoints say. it('moves from a refused native wire onto a translated one', async () => { - const callChatCompletions = vi.fn(async () => ({ + const callOpenAIChatCompletions = vi.fn(async () => ({ ok: false as const, modelKey: 'k', response: Response.json({ error: { message: 'slow down' } }, { status: 429 }), })); - const callMessages = vi.fn(async () => anthropicMessagesTurn('second')); + const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('second')); resolves([ - candidate('up_a', { chatCompletions: {} }, { callChatCompletions }), - candidate('up_b', { messages: {} }, { callMessages }), + candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions }), + candidate('up_b', { anthropicMessages: {} }, { callAnthropicMessages }), ]); - const { facts, drain } = await serveChatCompletions(); + const { facts, drain } = await serveOpenAIChatCompletions(); - expect(callChatCompletions).toHaveBeenCalledTimes(1); - expect(callMessages).toHaveBeenCalledTimes(1); + expect(callOpenAIChatCompletions).toHaveBeenCalledTimes(1); + expect(callAnthropicMessages).toHaveBeenCalledTimes(1); expect(facts['response.http.status']).toBe(200); expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('second'); await drain(); @@ -385,9 +385,9 @@ describe('the fork over wires', () => { // it must auto-compact, and an OpenAI-shaped upstream states the same condition in its own // words. The pair owns the translation of that refusal, and the handoff is what asks it. it('rewrites a context-window refusal into the pair-s own envelope', async () => { - resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions: contextExceeded })]); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions: contextExceeded })]); - const { facts, drain } = await serveMessages(); + const { facts, drain } = await serveAnthropicMessages(); expect(facts['response.http.status']).toBe(400); expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ @@ -403,12 +403,12 @@ describe('the fork over wires', () => { // what the pair exists to prevent. What crosses is the status and the sentence; the client's // own edge writes the shape. it('carries a refusal the pair does not speak about without its foreign envelope', async () => { - const callChatCompletions = vi.fn(async () => ({ + const callOpenAIChatCompletions = vi.fn(async () => ({ ok: false as const, modelKey: 'k', response: Response.json({ error: { message: 'no' } }, { status: 403 }), })); - resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions })]); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); - const { facts, drain } = await serveMessages(); + const { facts, drain } = await serveAnthropicMessages(); expect(facts['response.http.status']).toBe(403); const rendered = facts['response.chat.anthropicMessages.rendered'] as { type: string; error: { type: string; message: string } }; @@ -423,9 +423,9 @@ describe('the fork over wires', () => { // The other half of the same rule: a rule that *can* speak carries the object across, because // by then it is in the client's own protocol. it('carries the envelope a pair did rewrite', async () => { - resolves([candidate('up_a', { chatCompletions: {} }, { callChatCompletions: contextExceeded })]); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions: contextExceeded })]); - const { facts, drain } = await serveMessages(); + const { facts, drain } = await serveAnthropicMessages(); expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ type: 'error', @@ -450,9 +450,9 @@ describe('a rule that speaks about one protocol-s wire', () => { } as unknown as AnthropicMessagesPayload; resolves([candidate( 'up_a', - { chatCompletions: {} }, + { openaiChatCompletions: {} }, { - callChatCompletions: async (_model, body) => { + callOpenAIChatCompletions: async (_model, body) => { sent = body as { messages: { role: string }[] }; return openaiChatCompletionsTurn('hello'); }, @@ -460,7 +460,7 @@ describe('a rule that speaks about one protocol-s wire', () => { ['rewrite-mid-conv-system-to-user'], )]); - const { drain } = await serveMessages(payload); + const { drain } = await serveAnthropicMessages(payload); expect(sent?.messages.map(message => message.role)).toEqual(['system', 'user']); await drain(); @@ -472,14 +472,14 @@ describe('a rule that speaks about one protocol-s wire', () => { // *source* chain would ask for nothing here, and this turn would bill zero. it('asks for the usage chunk on a turn that arrived from another protocol', async () => { let sent: { stream_options?: unknown } | undefined; - resolves([candidate('up_a', { chatCompletions: {} }, { - callChatCompletions: async (_model, body) => { + resolves([candidate('up_a', { openaiChatCompletions: {} }, { + callOpenAIChatCompletions: async (_model, body) => { sent = body as { stream_options?: unknown }; return openaiChatCompletionsTurn('hello'); }, })]); - const { drain } = await serveMessages(); + const { drain } = await serveAnthropicMessages(); expect(sent?.stream_options).toEqual({ include_usage: true }); await drain(); @@ -490,14 +490,14 @@ describe('a rule that speaks about one protocol-s wire', () => { // the wire it landed on. A rule left in the Messages source chain would scrub nothing here. it('applies the target wire-s unguarded rules to a turn that arrived translated', async () => { let sent: { system?: unknown } | undefined; - resolves([candidate('up_a', { messages: {} }, { - callMessages: async (_model, body) => { + resolves([candidate('up_a', { anthropicMessages: {} }, { + callAnthropicMessages: async (_model, body) => { sent = body as { system?: unknown }; return anthropicMessagesTurn('hello'); }, }, ['strip-billing-attribution'])]); - const { drain } = await serveChatCompletions({ + const { drain } = await serveOpenAIChatCompletions({ ...openaiChatCompletionsPayload, messages: [ { role: 'system', content: 'be brief\nx-anthropic-billing-header: acct-42' }, @@ -511,18 +511,18 @@ describe('a rule that speaks about one protocol-s wire', () => { // And a rule the *other* wire owns stays off this turn. `stream_options` is a Chat // Completions field, so a Chat Completions turn dialled over Messages must not carry it — - // which is what the interceptor form said with `ctx.targetApi !== 'chat-completions'` and + // which is what the interceptor form said with `ctx.targetApi !== 'openaiChatCompletions'` and // what position says here. it('does not carry another wire-s request rules onto the wire it dialled', async () => { let sent: Record | undefined; - resolves([candidate('up_a', { messages: {} }, { - callMessages: async (_model, body) => { + resolves([candidate('up_a', { anthropicMessages: {} }, { + callAnthropicMessages: async (_model, body) => { sent = body as Record; return anthropicMessagesTurn('hello'); }, })]); - const { drain } = await serveChatCompletions(); + const { drain } = await serveOpenAIChatCompletions(); expect(sent).toBeDefined(); expect('stream_options' in sent!).toBe(false); diff --git a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts index 6b89b364df..0639611f12 100644 --- a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts @@ -32,7 +32,7 @@ const upstream = (models: Parameters[0]) => async (request // A `functionResponse` part in model content: a shape Gemini defines and neither target protocol // has anywhere to put, so the translator refuses it rather than dropping it. -const untranslatableGeminiBody = { +const untranslatableGeminiGenerateContentBody = { contents: [ { role: 'model', parts: [{ functionResponse: { name: 'f', response: { ok: true } } }] }, { role: 'user', parts: [{ text: 'hi' }] }, @@ -60,7 +60,7 @@ test('a body the target protocol cannot carry is refused in the client own proto await assertGoogleRpcRefusal(await requestApp('/v1beta/models/gpt-translate:generateContent', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify(untranslatableGeminiBody), + body: JSON.stringify(untranslatableGeminiGenerateContentBody), })); }, ); @@ -80,7 +80,7 @@ test('a body the counting wire cannot carry is refused in the client own protoco await assertGoogleRpcRefusal(await requestApp('/v1beta/models/claude-count:countTokens', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify(untranslatableGeminiBody), + body: JSON.stringify(untranslatableGeminiGenerateContentBody), })); }, ); diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 1486b008bd..87b4a21a91 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -7,34 +7,34 @@ import { describe, expect, it } from 'vitest'; import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; -import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/pipeline.ts'; -import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; -import { geminiGenerateContentCountTokensPipeline } from '../../../src/data-plane/chat/gemini-generate-content/count-tokens.ts'; -import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; import { anthropicMessagesCountTokensPipeline } from '../../../src/data-plane/chat/anthropic-messages/count-tokens.ts'; import { anthropicMessagesServePipeline } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; +import { geminiGenerateContentCountTokensPipeline } from '../../../src/data-plane/chat/gemini-generate-content/count-tokens.ts'; +import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; +import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; import { openaiResponsesCompactPipeline } from '../../../src/data-plane/chat/openai-responses/compact.ts'; import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; -import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; -import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; -import { imagesServePipeline } from '../../../src/data-plane/images/pipeline.ts'; +import { openaiAudioTranscriptionServePipeline } from '../../../src/data-plane/openai-audio/pipeline.ts'; +import { openaiCompletionsServePipeline } from '../../../src/data-plane/openai-completions/pipeline.ts'; +import { openaiEmbeddingsServePipeline } from '../../../src/data-plane/openai-embeddings/pipeline.ts'; +import { openaiImagesServePipeline } from '../../../src/data-plane/openai-images/pipeline.ts'; import { rerankServePipeline } from '../../../src/data-plane/rerank/pipeline.ts'; const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] = [ - ['embeddings', () => embeddingsServePipeline], + ['OpenAI Embeddings', () => openaiEmbeddingsServePipeline], ['rerank', () => rerankServePipeline({ sourceProtocol: 'cohere-v2', raw: {}, query: 'q', documents: ['a'] } as never)], - ['images generations', () => imagesServePipeline({ operation: 'generations', parameters: {} } as never)], - ['images edits', () => imagesServePipeline({ operation: 'edits', images: [], parameters: {} } as never)], - ['completions', () => completionsServePipeline], - ['audio transcription', () => audioTranscriptionServePipeline], + ['OpenAI Images generations', () => openaiImagesServePipeline({ operation: 'generations', parameters: {} } as never)], + ['OpenAI Images edits', () => openaiImagesServePipeline({ operation: 'edits', images: [], parameters: {} } as never)], + ['OpenAI Completions', () => openaiCompletionsServePipeline], + ['OpenAI Audio Transcriptions', () => openaiAudioTranscriptionServePipeline], ['alpha search', () => searchServePipeline({ kind: 'search' } as never)], - ['chat completions', () => openaiChatCompletionsServePipeline({ model: 'm', messages: [] } as never)], - ['messages', () => anthropicMessagesServePipeline({ model: 'm', messages: [] } as never)], - ['gemini', () => geminiGenerateContentServePipeline({ model: 'm', contents: [] } as never)], - ['responses', () => openaiResponsesServePipeline({ model: 'm', input: [] } as never)], - ['messages count_tokens', () => anthropicMessagesCountTokensPipeline({ model: 'm', messages: [] } as never)], - ['gemini countTokens', () => geminiGenerateContentCountTokensPipeline({ model: 'm', contents: [] } as never)], - ['responses compact', () => openaiResponsesCompactPipeline({ model: 'm', input: [] } as never)], + ['OpenAI Chat Completions', () => openaiChatCompletionsServePipeline({ model: 'm', messages: [] } as never)], + ['Anthropic Messages', () => anthropicMessagesServePipeline({ model: 'm', messages: [] } as never)], + ['Gemini generateContent', () => geminiGenerateContentServePipeline({ model: 'm', contents: [] } as never)], + ['OpenAI Responses', () => openaiResponsesServePipeline({ model: 'm', input: [] } as never)], + ['Anthropic Messages count_tokens', () => anthropicMessagesCountTokensPipeline({ model: 'm', messages: [] } as never)], + ['Gemini countTokens', () => geminiGenerateContentCountTokensPipeline({ model: 'm', contents: [] } as never)], + ['OpenAI Responses compact', () => openaiResponsesCompactPipeline({ model: 'm', input: [] } as never)], ]; describe('every family assembles', () => { diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/attempt.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/attempt.ts deleted file mode 100644 index 133b735faf..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/attempt.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { anthropicMessagesInterceptors, anthropicMessagesCountTokensInterceptors } from './interceptors/index.ts'; -import type { AnthropicMessagesInvocation } from './interceptors/types.ts'; -import { createAnthropicMessagesBillableUsageReader } from './usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; -import { openaiChatCompletionsAttempt } from '../openai-chat-completions/attempt.ts'; -import { openaiResponsesAttempt } from '../openai-responses/attempt.ts'; -import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerStreamResultToExecuteResult } from '../shared/provider-stream-result.ts'; -import { plainResultFromResponse } from '../shared/respond.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ModelCandidate, ExecuteResult, AnthropicMessagesUpstreamCallOptions, PlainResult } from '@floway-dev/provider'; -import { providerModelOf } from '@floway-dev/provider'; -import { translateAnthropicMessagesViaOpenAIChatCompletions, translateAnthropicMessagesViaOpenAIResponses } from '@floway-dev/translate'; - -// `/v1/messages` generate prefers a native Anthropic Messages target, then the -// translated OpenAI Responses path, then the translated OpenAI Chat Completions path. -export const anthropicMessagesGenerateTarget = chatTargetPicker(['anthropicMessages', 'openaiResponses', 'openaiChatCompletions']); - -// `count_tokens` has no translation path — only a native Anthropic Messages target -// satisfies the operation. -export const anthropicMessagesCountTokensTarget = chatTargetPicker(['anthropicMessages']); - -export interface AnthropicMessagesAttemptArgs { - readonly payload: AnthropicMessagesPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; - readonly anthropicBeta: readonly string[]; -} - -const buildAnthropicMessagesUpstreamCallOptions = ( - candidate: ModelCandidate, - ctx: ChatGatewayCtx, - headers: Headers, - anthropicBeta: readonly string[], -): AnthropicMessagesUpstreamCallOptions => ({ - ...buildUpstreamCallOptions(candidate, ctx, headers), - anthropicBeta, -}); - -export const anthropicMessagesAttempt = { - generate: async (args: AnthropicMessagesAttemptArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders, anthropicBeta } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - headers.delete('anthropic-beta'); - const targetApi = anthropicMessagesGenerateTarget.pick(candidate.model.endpoints); - const invocation: AnthropicMessagesInvocation = { - payload, - candidate, - targetApi, - headers, - }; - return await runInterceptors(invocation, ctx, anthropicMessagesInterceptors, async () => { - if (targetApi === 'anthropicMessages') { - if (candidate.rules !== undefined) applyRulesToUpstreamAnthropicMessages(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const providerResult = await candidate.provider.instance.callAnthropicMessages( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildAnthropicMessagesUpstreamCallOptions(candidate, ctx, invocation.headers, anthropicBeta), - ); - return await providerStreamResultToExecuteResult(providerResult, candidate, targetApi, ctx, createAnthropicMessagesBillableUsageReader()); - } - if (targetApi === 'openaiResponses') { - return await traverseTranslation( - invocation.payload, - p => translateAnthropicMessagesViaOpenAIResponses(p, { model: candidate.model.id }), - translated => openaiResponsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - if (targetApi === 'openaiChatCompletions') { - return await traverseTranslation( - invocation.payload, - p => translateAnthropicMessagesViaOpenAIChatCompletions(p, { model: candidate.model.id }), - translated => openaiChatCompletionsAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - throw new Error(`anthropicMessagesAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, - - countTokens: async (args: AnthropicMessagesAttemptArgs): Promise => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders, anthropicBeta } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - headers.delete('anthropic-beta'); - // `pick` here is contractually total — serve filtered with - // `anthropicMessagesCountTokensTarget.canServe`, so a non-anthropic-messages candidate is - // a contract breach. - const targetApi = anthropicMessagesCountTokensTarget.pick(candidate.model.endpoints); - const invocation: AnthropicMessagesInvocation = { - payload, - candidate, - targetApi, - headers, - }; - const response = await runInterceptors(invocation, ctx, anthropicMessagesCountTokensInterceptors, async () => { - if (candidate.rules !== undefined) applyRulesToUpstreamAnthropicMessages(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const { response } = await candidate.provider.instance.callAnthropicMessagesCountTokens( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildAnthropicMessagesUpstreamCallOptions(candidate, ctx, invocation.headers, anthropicBeta), - ); - return response; - }); - return await plainResultFromResponse(response, candidate.provider.upstreamId); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index 5a3a60a9e7..0221606b21 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -1,12 +1,12 @@ // `/v1/messages/count_tokens` as a pipeline. // -// emitMessagesTokenCount the edge: writes the measurement, or the refusal, as one body +// emitAnthropicMessagesTokenCount the edge: writes the measurement, or the refusal, as one body // writeSettlement above the fork, so a measurement is sampled once however many it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record // the three request rules generation applies, and the web-search request preparation -// callMessagesCountTokensUpstream the ending: measures, and never opens a stream +// callAnthropicMessagesCountTokensUpstream the ending: measures, and never opens a stream // // A second **operation** over this protocol rather than another wire under // `anthropicMessagesServePipeline`, which is why it is a chain of its own rather than a branch inside @@ -27,9 +27,9 @@ // that asked for the native server tool is measured on the client-tool shape the gateway would // actually put on the wire. -import { analyzeMessagesAffinity } from './affinity/ingress.ts'; -import { renderMessagesError } from './errors.ts'; -import { prepareMessagesWebSearchShimRequest } from './interceptors/web-search-shim.ts'; +import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; +import { renderAnthropicMessagesError } from './errors.ts'; +import { prepareAnthropicMessagesWebSearchShimRequest } from './interceptors/web-search-shim.ts'; import type { AnthropicMessagesFacts } from './pipeline.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; @@ -39,11 +39,11 @@ import { failover } from '../../pipeline/stages.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import { - applyRoleCompatibilityToMessages, - disableReasoningOnForcedToolChoiceForMessages, - stripBillingAttributionFromMessages, + applyRoleCompatibilityToAnthropicMessages, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + stripBillingAttributionFromAnthropicMessages, } from '../interceptors.ts'; -import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; +import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; @@ -53,7 +53,7 @@ import { providerModelOf } from '@floway-dev/provider'; /** Counting has no translation path: only an upstream's own Messages endpoint can answer the * question, so a candidate that would serve generation over a translated wire cannot serve * this. */ -export const anthropicMessagesCountTokensTarget = chatTargetPicker(['messages']); +export const anthropicMessagesCountTokensTarget = chatTargetPicker(['anthropicMessages']); /** What a measurement hands up. It rides at a protocol's own response key — which is what * lets the request rules generation runs be the same stages here — and the arm is narrowed @@ -69,14 +69,14 @@ type M = { [P in K]: AnthropicMessagesFa * to stream: what this decides is only whether the client is shown the upstream's counts or * the refusal that stands in for them. */ -const emitMessagesTokenCount = defineStage< +const emitAnthropicMessagesTokenCount = defineStage< Record, Record, Counted<'response.chat.anthropicMessages'> & M<'response.http.headers'>, M<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ - name: 'emitMessagesTokenCount', + name: 'emitAnthropicMessagesTokenCount', through: { request: { needs: [], consumes: [], provides: [] }, response: { @@ -98,7 +98,7 @@ const emitMessagesTokenCount = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.anthropicMessages.rendered': move(renderFailure(answer, () => renderMessagesError(answer.status, answer.message))), + 'response.chat.anthropicMessages.rendered': move(renderFailure(answer, () => renderAnthropicMessagesError(answer.status, answer.message))), 'response.http.status': answer.status, }; } @@ -127,7 +127,7 @@ const emitMessagesTokenCount = defineStage< * A tool definition this protocol rejects is an answer it already holds, which is why it * carries the `return` trait. */ -const prepareMessagesWebSearchRequest = defineStage< +const prepareAnthropicMessagesWebSearchRequest = defineStage< M<'request.chat.anthropicMessages' | 'route.attempt'>, M<'request.chat.anthropicMessages'>, Counted<'response.chat.anthropicMessages'>, @@ -135,7 +135,7 @@ const prepareMessagesWebSearchRequest = defineStage< Counted<'response.chat.anthropicMessages'> & M<'response.usage.billable' | 'response.http.headers'>, ChatServices >({ - name: 'prepareMessagesWebSearchRequest', + name: 'prepareAnthropicMessagesWebSearchRequest', through: { request: { needs: ['request.chat.anthropicMessages', 'route.attempt'], @@ -147,7 +147,7 @@ const prepareMessagesWebSearchRequest = defineStage< return: { provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers'] }, execute: async (facts, next) => { if (!facts['route.attempt'].flags.includes('messages-web-search-shim')) return await next(facts); - const prepared = prepareMessagesWebSearchShimRequest(facts['request.chat.anthropicMessages']); + const prepared = prepareAnthropicMessagesWebSearchShimRequest(facts['request.chat.anthropicMessages']); if (prepared.type === 'invalid-request') { // Nothing was dialled, which is what an empty billed set and an empty header list say // on the other two keys. @@ -170,12 +170,12 @@ const prepareMessagesWebSearchRequest = defineStage< * value — the one shape this operation ever produces, because a measurement is a body and * never a stream. */ -const callMessagesCountTokensUpstream = defineStage< +const callAnthropicMessagesCountTokensUpstream = defineStage< M<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, Counted<'response.chat.anthropicMessages'> & M<'response.usage.billable' | 'response.http.headers'>, ChatServices >({ - name: 'callMessagesCountTokensUpstream', + name: 'callAnthropicMessagesCountTokensUpstream', return: { provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers'], }, @@ -185,12 +185,12 @@ const callMessagesCountTokensUpstream = defineStage< // The payload affinity materialized for this candidate, as every stage between the fork // and here has rewritten it — built for the dial exactly as generation builds it, so what // is measured is what generation would be charged for. - const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamMessages); + const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamAnthropicMessages); // Anthropic's beta flags have a typed path of their own, so no header allowlist can admit // them, and they are the client's own only when the client spoke this protocol. const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); - const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'messages' + const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'anthropicMessages' ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) : []; headers.delete('anthropic-beta'); @@ -201,7 +201,7 @@ const callMessagesCountTokensUpstream = defineStage< let response: Response; try { - ({ response } = await candidate.provider.instance.callMessagesCountTokens( + ({ response } = await candidate.provider.instance.callAnthropicMessagesCountTokens( providerModelOf(candidate), body, use.gateway.abortSignal, @@ -257,11 +257,11 @@ const callMessagesCountTokensUpstream = defineStage< * seen. */ export const anthropicMessagesCountTokensWire: readonly Stage[] = [ - prepareMessagesWebSearchRequest, - stripBillingAttributionFromMessages, - disableReasoningOnForcedToolChoiceForMessages, - applyRoleCompatibilityToMessages, - callMessagesCountTokensUpstream, + prepareAnthropicMessagesWebSearchRequest, + stripBillingAttributionFromAnthropicMessages, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + applyRoleCompatibilityToAnthropicMessages, + callAnthropicMessagesCountTokensUpstream, ]; /** A candidate that cannot serve *this* request is not a candidate — and what the client's @@ -269,7 +269,7 @@ export const anthropicMessagesCountTokensWire: readonly Stage[] = [ * built from the request rather than being a constant. */ const narrowing = (payload: AnthropicMessagesPayload): ChatNarrowing> => ({ canServe: candidate => anthropicMessagesCountTokensTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeMessagesAffinity(payload, gateway.affinity.codec), + affinity: async gateway => await analyzeAnthropicMessagesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /messages/count_tokens endpoint.`, refuse: (status, message) => ({ 'response.chat.anthropicMessages': { status, message } }), refuses: ['response.chat.anthropicMessages'], @@ -285,7 +285,7 @@ export type AnthropicMessagesCountTokensExit = M< export const anthropicMessagesCountTokensPipeline = (payload: AnthropicMessagesPayload): Pipeline => compose('anthropicMessagesCountTokens', [ - emitMessagesTokenCount, + emitAnthropicMessagesTokenCount, // A measurement goes through settlement like every other run. It provides an empty // billed set because nothing here is billable today, not because the operation is // exempt — an upstream that began charging for it would provide a non-empty one and diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts index 993434ddbe..2ea9c99459 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts @@ -8,7 +8,7 @@ import { generateAnthropicId } from '@floway-dev/protocols/anthropic-messages'; * Anthropic's own — `type` at the top, the error's own `type` inside, and a `request_id` a * caller quotes back in a support thread — so it is the protocol's shape rather than the * gateway's, and an OpenAI-shaped one would carry none of those fields. */ -export const renderMessagesError = (status: number, message: string): Record => ({ +export const renderAnthropicMessagesError = (status: number, message: string): Record => ({ type: 'error', error: { type: anthropicErrorTypeForStatus(status), message }, request_id: generateAnthropicId('req'), diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts index 4eef5bfb47..505d2c62c5 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts @@ -7,13 +7,13 @@ // stages. import { anthropicMessagesCountTokensPipeline } from './count-tokens.ts'; -import { renderMessagesError } from './errors.ts'; +import { renderAnthropicMessagesError } from './errors.ts'; import { anthropicMessagesKeepAlive, anthropicMessagesServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; -import { createNonResponsesSourceStore } from '../openai-responses/items/store.ts'; +import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; import { move } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; @@ -62,7 +62,7 @@ export const anthropicMessagesHttp = { const ingress = await readIngress(c); const request = readRequest(ingress.body.bytes); if (request.type === 'invalid') { - return refuse(c, ingress, Response.json(renderMessagesError(400, request.message), { status: 400 })); + return refuse(c, ingress, Response.json(renderAnthropicMessagesError(400, request.message), { status: 400 })); } const { payload } = request; @@ -73,7 +73,7 @@ export const anthropicMessagesHttp = { const prologue = openChatPrologue(c, ingress, { wantsStream, model: payload.model, - storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), }); return await serveThrough( @@ -82,7 +82,7 @@ export const anthropicMessagesHttp = { anthropicMessagesServePipeline(payload), move({ 'ingress.http.headers': prologue.headers, - 'ingress.chat.sourceProtocol': 'messages', + 'ingress.chat.sourceProtocol': 'anthropicMessages', 'ingress.chat.anthropicMessages.wantsStream': wantsStream, 'request.chat.anthropicMessages': payload, 'serve.model': payload.model, @@ -102,7 +102,7 @@ export const anthropicMessagesHttp = { const ingress = await readIngress(c); const request = readRequest(ingress.body.bytes); if (request.type === 'invalid') { - return refuse(c, ingress, Response.json(renderMessagesError(400, request.message), { status: 400 })); + return refuse(c, ingress, Response.json(renderAnthropicMessagesError(400, request.message), { status: 400 })); } const { payload } = request; @@ -112,7 +112,7 @@ export const anthropicMessagesHttp = { const prologue = openChatPrologue(c, ingress, { wantsStream: false, model: payload.model, - storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), }); return await serveThrough( @@ -121,7 +121,7 @@ export const anthropicMessagesHttp = { anthropicMessagesCountTokensPipeline(payload), move({ 'ingress.http.headers': prologue.headers, - 'ingress.chat.sourceProtocol': 'messages', + 'ingress.chat.sourceProtocol': 'anthropicMessages', 'request.chat.anthropicMessages': payload, 'serve.model': payload.model, }) as never, diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 5c3b7b6d30..4fdfa430c8 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -1,6 +1,6 @@ // Messages as a pipeline, on the chain Chat Completions established. // -// emitMessages the edge: writes the answer in the shape the client asked for +// emitAnthropicMessages the edge: writes the answer in the shape the client asked for // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate @@ -20,11 +20,11 @@ // nothing. Only its request half survives on a live path, in the counting chain. The code is kept // so that porting it starts from something rather than from nothing. -import { wrapMessagesAffinityEgress } from './affinity/egress.ts'; -import { analyzeMessagesAffinity } from './affinity/ingress.ts'; -import { renderMessagesError } from './errors.ts'; +import { wrapAnthropicMessagesAffinityEgress } from './affinity/egress.ts'; +import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; +import { renderAnthropicMessagesError } from './errors.ts'; import { isClaudeCodeProbe, probeFrames } from './interceptors/answer-claude-code-probe.ts'; -import { createMessagesBillableUsageReader } from './usage.ts'; +import { createAnthropicMessagesBillableUsageReader } from './usage.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; @@ -40,32 +40,32 @@ import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.t import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { - applyRoleCompatibilityToMessages, - disableReasoningOnForcedToolChoiceForMessages, - stripBillingAttributionFromMessages, + applyRoleCompatibilityToAnthropicMessages, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + stripBillingAttributionFromAnthropicMessages, } from '../interceptors.ts'; import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { applyRulesToUpstreamMessages } from '../shared/alias-rules.ts'; +import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; import { sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { - collectMessagesProtocolEventsToResult, + collectAnthropicMessagesProtocolEventsToResult, anthropicMessagesProtocolFrameToSSEFrame, - MESSAGES_MISSING_TERMINAL_MESSAGE, + ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, parseAnthropicBetaHeader, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent, } from '@floway-dev/protocols/anthropic-messages'; import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; -import { translateMessagesViaChatCompletions, translateMessagesViaResponses } from '@floway-dev/translate'; +import { translateAnthropicMessagesViaOpenAIChatCompletions, translateAnthropicMessagesViaOpenAIResponses } from '@floway-dev/translate'; /** `/v1/messages` prefers its own wire, then the translated Responses path, then the * translated Chat Completions path. */ -export const anthropicMessagesTarget = chatTargetPicker(['messages', 'responses', 'chat-completions']); +export const anthropicMessagesTarget = chatTargetPicker(['anthropicMessages', 'openaiResponses', 'openaiChatCompletions']); /** What this family adds to the chat space. */ export interface AnthropicMessagesFacts extends ChatFacts { @@ -89,14 +89,14 @@ type M = { [P in K]: AnthropicMessagesFa * reassembly. Neither shape can answer with half a message: the ending fails a stream that * ran out before `message_stop`, whichever of the two the client asked for. */ -const emitMessages = defineStage< +const emitAnthropicMessages = defineStage< M<'ingress.chat.anthropicMessages.wantsStream'>, M<'ingress.chat.anthropicMessages.wantsStream'>, M<'ingress.chat.anthropicMessages.wantsStream' | 'response.chat.anthropicMessages' | 'response.http.headers'>, M<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ - name: 'emitMessages', + name: 'emitAnthropicMessages', through: { request: { needs: ['ingress.chat.anthropicMessages.wantsStream'], consumes: [], provides: [] }, response: { @@ -118,7 +118,7 @@ const emitMessages = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.anthropicMessages.rendered': move(renderFailure(answer, () => renderMessagesError(answer.status, answer.message))), + 'response.chat.anthropicMessages.rendered': move(renderFailure(answer, () => renderAnthropicMessagesError(answer.status, answer.message))), 'response.http.status': answer.status, }; } @@ -136,7 +136,7 @@ const emitMessages = defineStage< // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. const frames = recordFrames( - wrapMessagesAffinityEgress( + wrapAnthropicMessagesAffinityEgress( answer.frames as AsyncIterable>, affinityEgressOptions(use.gateway), ), @@ -147,7 +147,7 @@ const emitMessages = defineStage< ...rest, 'response.http.headers': forClient, 'response.chat.anthropicMessages.rendered': move( - await collectMessagesProtocolEventsToResult(frames) as unknown as Record, + await collectAnthropicMessagesProtocolEventsToResult(frames) as unknown as Record, ), 'response.http.status': 200, }; @@ -252,12 +252,12 @@ const answerClaudeCodeProbe = defineStage< }, }); -const callMessagesUpstream = (streamedUsage: string) => defineStage< +const callAnthropicMessagesUpstream = (streamedUsage: string) => defineStage< M<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, M<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ - name: 'callMessagesUpstream', + name: 'callAnthropicMessagesUpstream', return: { provides: [ 'response.chat.anthropicMessages', @@ -276,7 +276,7 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< // every stage between the fork and here has rewritten it — or, on a translated wire, what // the handoff put here. A thinking signature was rewritten for the upstream that will see // it, and one that no upstream but the issuer can read was dropped rather than sent on. - const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamMessages); + const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamAnthropicMessages); // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump @@ -286,14 +286,14 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< // a translation asked for nothing on this wire, so the field is read off and then dropped // whichever protocol sent it. const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); - const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'messages' + const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'anthropicMessages' ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) : []; headers.delete('anthropic-beta'); let result; try { - result = await candidate.provider.instance.callMessages( + result = await candidate.provider.instance.callAnthropicMessages( providerModelOf(candidate), body, use.gateway.abortSignal, @@ -338,7 +338,7 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< // This candidate answered, so it is the one a follow-up turn carrying our own state // must come back to. use.selectAffinity(candidate); - const metered = meterMessages(result.events, identity, use.gateway.attempt); + const metered = meterAnthropicMessages(result.events, identity, use.gateway.attempt); return move({ ...facts, 'response.chat.anthropicMessages': { kind: 'stream' as const, frames: metered.frames }, @@ -359,10 +359,10 @@ const callMessagesUpstream = (streamedUsage: string) => defineStage< * that leaves for another protocol. */ export const anthropicMessagesWire = (streamedUsage: string): readonly Stage[] => [ - stripBillingAttributionFromMessages, - disableReasoningOnForcedToolChoiceForMessages, - applyRoleCompatibilityToMessages, - callMessagesUpstream(streamedUsage), + stripBillingAttributionFromAnthropicMessages, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + applyRoleCompatibilityToAnthropicMessages, + callAnthropicMessagesUpstream(streamedUsage), ]; /** This family's own reading, which every wire under it hands up. */ @@ -372,23 +372,23 @@ const STREAMED_USAGE = 'response.chat.anthropicMessages.streamedUsage'; * one is a handoff and then the target protocol's own wire. */ const anthropicMessagesWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { switch (target) { - case 'messages': + case 'anthropicMessages': return compose('anthropicMessagesNative', anthropicMessagesWire(STREAMED_USAGE)); - case 'responses': + case 'openaiResponses': return compose('anthropicMessagesViaOpenAIResponses', [ handOff({ from: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, - trip: async payload => await translateMessagesViaResponses(payload, { model: candidate.model.id }), + trip: async payload => await translateAnthropicMessagesViaOpenAIResponses(payload, { model: candidate.model.id }), }), ...openaiResponsesWire(STREAMED_USAGE), ]); - case 'chat-completions': + case 'openaiChatCompletions': return compose('anthropicMessagesViaOpenAIChatCompletions', [ handOff({ from: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, - trip: async payload => await translateMessagesViaChatCompletions(payload, { model: candidate.model.id }), + trip: async payload => await translateAnthropicMessagesViaOpenAIChatCompletions(payload, { model: candidate.model.id }), }), ...openaiChatCompletionsWire(STREAMED_USAGE), ]); @@ -399,7 +399,7 @@ const anthropicMessagesWireFor = (target: ChatTargetApi, candidate: ModelCandida * pass and the client's stream is what drives it. Anthropic states input accounting on * `message_start` and output accounting on `message_delta`, so the reader that merges the * two is per-stream state and is made here rather than shared. */ -const meterMessages = ( +const meterAnthropicMessages = ( source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, @@ -409,14 +409,14 @@ const meterMessages = ( // Running out without the terminal frame is what "it did not finish" means, and it is known // at the same moment the usage is. let sawTerminal = false; - const readBillableUsage = createMessagesBillableUsageReader(); + const readBillableUsage = createAnthropicMessagesBillableUsageReader(); const generator = (async function* () { let reported: BillableUsage | undefined; try { for await (const frame of source) { // Time to first token is measured where the token is, which is the only place that // knows a frame carries generated content rather than the envelope around it. - if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'messages')) { + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'anthropicMessages')) { attempt.firstOutputTokenAt = performance.now(); } if (frame.type === 'event') { @@ -427,11 +427,11 @@ const meterMessages = ( // The turn is over, so there is nothing further to read. An upstream that holds the // connection open past `message_stop` would otherwise hold the client's stream open // with it; returning here closes the read, which cancels the upstream. - if (isMessagesTerminalFrame(frame)) { sawTerminal = true; return; } + if (isAnthropicMessagesTerminalFrame(frame)) { sawTerminal = true; return; } } // Frames ran out with no terminal event, which is a turn nobody can answer from: the // message was never stopped and never failed. - throw new Error(MESSAGES_MISSING_TERMINAL_MESSAGE); + throw new Error(ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE); } finally { // Reached however the frames ended — the terminal event, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are @@ -445,7 +445,7 @@ const meterMessages = ( /** What ends a Messages turn. Anthropic's own stream terminator is an event rather than a * transport sentinel, and a stream that failed mid-turn says so with `error` in place of the * `message_stop` that will now never come. */ -const isMessagesTerminalFrame = (frame: ProtocolFrame): boolean => +const isAnthropicMessagesTerminalFrame = (frame: ProtocolFrame): boolean => frame.type === 'event' && (frame.event.type === 'message_stop' || frame.event.type === 'error'); /** What one attempt is billable for. An upstream that reported nothing leaves no quantities @@ -464,7 +464,7 @@ const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModel * built from the request rather than being a constant. */ const narrowing = (payload: AnthropicMessagesPayload): ChatNarrowing> => ({ canServe: candidate => anthropicMessagesTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeMessagesAffinity(payload, gateway.affinity.codec), + affinity: async gateway => await analyzeAnthropicMessagesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /messages endpoint.`, refuse: (status, message) => ({ 'response.chat.anthropicMessages': { status, message }, @@ -492,7 +492,7 @@ export type AnthropicMessagesServeExit = M< export const anthropicMessagesServePipeline = (payload: AnthropicMessagesPayload): Pipeline => compose('anthropicMessagesServe', [ - emitMessages, + emitAnthropicMessages, writeSettlement( handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), handedUp => (handedUp as { 'response.chat.anthropicMessages.streamedUsage'?: unknown })['response.chat.anthropicMessages.streamedUsage'] !== null, diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts index 8bb413ccb8..f4d18cab64 100644 --- a/packages/gateway/src/data-plane/chat/facts.ts +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -19,7 +19,7 @@ import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-respon /** The four protocols a client can speak here. Gemini is source-only: nothing translates * into it, so by the ruling it contributes one pipeline rather than two — a source role * and no target role. */ -export type ChatSourceProtocol = 'chatCompletions' | 'messages' | 'responses' | 'gemini'; +export type ChatSourceProtocol = 'openaiChatCompletions' | 'anthropicMessages' | 'openaiResponses' | 'geminiGenerateContent'; export interface ChatFacts extends GatewayFacts { /** Which protocol the client spoke. Read by the stages that must know — a vendor diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/attempt.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/attempt.ts deleted file mode 100644 index 9a90fc39c5..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/attempt.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { geminiGenerateContentStatusForHttpStatus } from './errors.ts'; -import { geminiGenerateContentCountTokensInterceptors, geminiGenerateContentInterceptors } from './interceptors/index.ts'; -import { stripUnsupportedPartFieldsFromPayload } from './interceptors/strip-unsupported-part-fields.ts'; -import { stripUnsupportedToolsFromPayload } from './interceptors/strip-unsupported-tools.ts'; -import { anthropicMessagesAttempt } from '../anthropic-messages/attempt.ts'; -import { openaiChatCompletionsAttempt } from '../openai-chat-completions/attempt.ts'; -import { openaiResponsesAttempt } from '../openai-responses/attempt.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import { type ModelCandidate, plainResult, type ExecuteResult, type GeminiGenerateContentInvocation, type PlainResult } from '@floway-dev/provider'; -import { translateGeminiGenerateContentViaOpenAIChatCompletions, translateGeminiGenerateContentViaAnthropicMessages, translateGeminiGenerateContentViaOpenAIResponses } from '@floway-dev/translate'; - -// Gemini generateContent has no native upstream target in the provider API; prefer OpenAI Chat Completions -// Completions, then Anthropic Messages, then OpenAI Responses for generate. countTokens has -// no translation path beyond native Anthropic Messages count_tokens. -export const geminiGenerateContentGenerateTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); -export const geminiGenerateContentCountTokensTarget = chatTargetPicker(['anthropicMessages']); - -export interface GeminiGenerateContentAttemptGenerateArgs { - readonly payload: GeminiGenerateContentPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export interface GeminiGenerateContentAttemptCountTokensArgs { - readonly payload: GeminiGenerateContentPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export const geminiGenerateContentAttempt = { - generate: async (args: GeminiGenerateContentAttemptGenerateArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = structuredClone(sourcePayload); - const headers = new Headers(sourceHeaders); - const targetApi = geminiGenerateContentGenerateTarget.pick(candidate.model.endpoints); - const invocation: GeminiGenerateContentInvocation = { payload, candidate, targetApi, headers }; - return await runInterceptors(invocation, ctx, geminiGenerateContentInterceptors, async () => { - // Gemini generateContent has no native upstream target today — every targetApi we - // pick is reached via translation. The dispatch threads each branch - // through `traverseTranslation` so each inner attempt owns its own - // interceptor chain and rewrite. - const transCtx = { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - }; - if (targetApi === 'anthropicMessages') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiGenerateContentViaAnthropicMessages(p, transCtx), - translated => anthropicMessagesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }), - ); - } - if (targetApi === 'openaiResponses') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiGenerateContentViaOpenAIResponses(p, transCtx), - translated => openaiResponsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - if (targetApi === 'openaiChatCompletions') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiGenerateContentViaOpenAIChatCompletions(p, transCtx), - translated => openaiChatCompletionsAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - throw new Error(`geminiGenerateContentAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, - - countTokens: async (args: GeminiGenerateContentAttemptCountTokensArgs): Promise => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = structuredClone(sourcePayload); - const headers = new Headers(sourceHeaders); - const targetApi = geminiGenerateContentCountTokensTarget.pick(candidate.model.endpoints); - const invocation: GeminiGenerateContentInvocation = { payload, candidate, targetApi, headers }; - return await runInterceptors(invocation, ctx, geminiGenerateContentCountTokensInterceptors, async () => { - // Gemini generateContent countTokens has no native upstream; translate to Anthropic Messages and - // delegate to `anthropicMessagesAttempt.countTokens`, then reshape the Anthropic Messages - // count_tokens reply into the Gemini generateContent `{ totalTokens }` envelope. The - // shipped Gemini generateContent interceptors that mutate the payload pre-dispatch - // cannot run via the countTokens interceptor list — the post-`run()` - // ones inspect event streams the result type cannot carry — so the - // payload-mutators are applied inline here before translation; the - // attempt-owned payload clone keeps the caller's source intact. - const transCtx = { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - }; - const cleaned = invocation.payload; - stripUnsupportedPartFieldsFromPayload(cleaned); - stripUnsupportedToolsFromPayload(cleaned); - delete cleaned.safetySettings; - const trip = await translateGeminiGenerateContentViaAnthropicMessages(cleaned, transCtx); - const { stream: _stream, ...target } = trip.target; - const anthropicMessagesResult = await anthropicMessagesAttempt.countTokens({ - payload: target, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }); - return reshapeAnthropicMessagesCountAsGeminiGenerateContent(anthropicMessagesResult); - }); - }, -}; - -// Reshape the Anthropic Messages count_tokens body into the Gemini generateContent `{ totalTokens }` -// envelope. The upstream body shape is provider-specific: Anthropic emits -// `{ input_tokens }`, Copilot's translated count emits `{ total_tokens }`; -// either is accepted. A missing or non-numeric figure is surfaced as a -// 502 Google-RPC error so the caller sees a typed Gemini generateContent failure rather -// than a passthrough of the upstream shape. -const reshapeAnthropicMessagesCountAsGeminiGenerateContent = (anthropicMessagesResult: PlainResult): PlainResult => { - if (anthropicMessagesResult.status !== 200) { - // Empty upstream bodies fall back to a fixed message so the Google-RPC envelope is never empty. - const text = new TextDecoder().decode(anthropicMessagesResult.body); - return geminiGenerateContentErrorPlainResult(anthropicMessagesResult.status, text || 'Upstream token counting request failed.', anthropicMessagesResult.upstreamId); - } - let decoded: unknown; - try { decoded = JSON.parse(new TextDecoder().decode(anthropicMessagesResult.body)); } catch {} - const upstreamTokenCounts = decoded && typeof decoded === 'object' - ? decoded as { input_tokens?: unknown; total_tokens?: unknown } - : {}; - const totalTokens = typeof upstreamTokenCounts.input_tokens === 'number' - ? upstreamTokenCounts.input_tokens - : typeof upstreamTokenCounts.total_tokens === 'number' - ? upstreamTokenCounts.total_tokens - : null; - if (totalTokens === null) { - return geminiGenerateContentInternalPlainResult(502, new Error('Invalid upstream token counting response.')); - } - return plainResult( - 200, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ totalTokens })), - anthropicMessagesResult.upstreamId, - ); -}; - -const geminiGenerateContentErrorPlainResult = (status: number, message: string, upstream?: string): PlainResult => plainResult( - status, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ error: { code: status, message, status: geminiGenerateContentStatusForHttpStatus(status) } })), - upstream, -); - -const geminiGenerateContentInternalPlainResult = (status: number, error: Error): PlainResult => plainResult( - status, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ - error: { - code: status, - message: error.message, - status: geminiGenerateContentStatusForHttpStatus(status), - type: 'internal_error', - name: error.name, - stack: error.stack, - }, - })), -); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index 5c0ddbfb5f..fe948448e1 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -1,12 +1,12 @@ // Gemini's `:countTokens` as a pipeline. // -// emitGeminiTokenCount the edge: writes the measurement, or the refusal, as one body +// emitGeminiGenerateContentTokenCount the edge: writes the measurement, or the refusal, as one body // writeSettlement above the fork, so a measurement is sampled once however many it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record // the three strippers the same protocol-shape cleanups generation applies -// measureGeminiAsMessages the pair: asks the question in Messages and reads the answer back +// measureGeminiGenerateContentAsAnthropicMessages the pair: asks the question in Messages and reads the answer back // the Messages count wire the ending, shared with `/v1/messages/count_tokens` // // A second **operation** over this protocol rather than another wire under @@ -25,8 +25,8 @@ // cannot be sent, so it goes at source. The thought suppressor does not — it rewrites a // stream, and a measurement has none. -import { analyzeGeminiAffinity } from './affinity/ingress.ts'; -import { renderGeminiError } from './errors.ts'; +import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; +import { renderGeminiGenerateContentError } from './errors.ts'; import type { GeminiGenerateContentFacts } from './pipeline.ts'; import { asJsonObject, readJsonNumber } from '../../../shared/json-helpers.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; @@ -34,20 +34,20 @@ import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import { - stripSafetySettingsFromGemini, - stripUnsupportedPartFieldsFromGemini, - stripUnsupportedToolsFromGemini, + stripSafetySettingsFromGeminiGenerateContent, + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, } from '../interceptors.ts'; import { anthropicMessagesCountTokensWire, type TokenCountAnswer } from '../anthropic-messages/count-tokens.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -import { translateGeminiViaMessages, TranslatorInputError } from '@floway-dev/translate'; +import { translateGeminiGenerateContentViaAnthropicMessages, TranslatorInputError } from '@floway-dev/translate'; /** Counting is reachable only over an upstream's own Messages endpoint: no other protocol * answers the question, and no translation invents an answer. */ -export const geminiGenerateContentCountTokensTarget = chatTargetPicker(['messages']); +export const geminiGenerateContentCountTokensTarget = chatTargetPicker(['anthropicMessages']); type Counted = { [P in K]: TokenCountAnswer }; @@ -58,14 +58,14 @@ type G = { [P in K]: GeminiGenerateC * stream: what this decides is only whether the client is shown the count or the refusal that * stands in for it. */ -const emitGeminiTokenCount = defineStage< +const emitGeminiGenerateContentTokenCount = defineStage< Record, Record, Counted<'response.chat.geminiGenerateContent'> & G<'response.http.headers'>, G<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ - name: 'emitGeminiTokenCount', + name: 'emitGeminiGenerateContentTokenCount', through: { request: { needs: [], consumes: [], provides: [] }, response: { @@ -87,7 +87,7 @@ const emitGeminiTokenCount = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.geminiGenerateContent.rendered': move(renderFailure(answer, () => renderGeminiError(answer.status, answer.message))), + 'response.chat.geminiGenerateContent.rendered': move(renderFailure(answer, () => renderGeminiGenerateContentError(answer.status, answer.message))), 'response.http.status': answer.status, }; } @@ -110,7 +110,7 @@ const emitGeminiTokenCount = defineStage< * It carries the `return` trait for the same reason a handoff does — a body the target * protocol cannot represent is answered here rather than measured. */ -const measureGeminiAsMessages = defineStage< +const measureGeminiGenerateContentAsAnthropicMessages = defineStage< G<'request.chat.geminiGenerateContent' | 'route.attempt'>, G<'request.chat.anthropicMessages'>, Counted<'response.chat.anthropicMessages'>, @@ -118,7 +118,7 @@ const measureGeminiAsMessages = defineStage< Counted<'response.chat.geminiGenerateContent'> & G<'response.usage.billable' | 'response.http.headers'>, ChatServices >({ - name: 'measureGeminiAsMessages', + name: 'measureGeminiGenerateContentAsAnthropicMessages', through: { request: { needs: ['request.chat.geminiGenerateContent', 'route.attempt'], @@ -138,7 +138,7 @@ const measureGeminiAsMessages = defineStage< let trip; try { - trip = await translateGeminiViaMessages(asked, { + trip = await translateGeminiGenerateContentViaAnthropicMessages(asked, { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, }); @@ -165,7 +165,7 @@ const measureGeminiAsMessages = defineStage< const back = await next({ ...down, 'request.chat.anthropicMessages': move(target) }); const { 'response.chat.anthropicMessages': counted, ...up } = back; - return { ...up, 'response.chat.geminiGenerateContent': move(asGeminiTokenCount(counted)) }; + return { ...up, 'response.chat.geminiGenerateContent': move(asGeminiGenerateContentTokenCount(counted)) }; }, }); @@ -177,7 +177,7 @@ const measureGeminiAsMessages = defineStage< * protocol calls `totalTokens`; a body carrying neither is an answer this gateway cannot * read, which is the gateway failing to serve rather than anything the client can fix. */ -const asGeminiTokenCount = (counted: TokenCountAnswer): TokenCountAnswer => { +const asGeminiGenerateContentTokenCount = (counted: TokenCountAnswer): TokenCountAnswer => { if (isFailure(counted)) { return { ...counted, @@ -197,7 +197,7 @@ const asGeminiTokenCount = (counted: TokenCountAnswer): TokenCountAnswer => { * built from the request rather than being a constant. */ const narrowing = (payload: GeminiGenerateContentPayload): ChatNarrowing> => ({ canServe: candidate => geminiGenerateContentCountTokensTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeGeminiAffinity(payload, gateway.affinity.codec), + affinity: async gateway => await analyzeGeminiGenerateContentAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support countTokens.`, refuse: (status, message) => ({ 'response.chat.geminiGenerateContent': { status, message } }), refuses: ['response.chat.geminiGenerateContent'], @@ -213,7 +213,7 @@ export type GeminiGenerateContentCountTokensExit = G< export const geminiGenerateContentCountTokensPipeline = (payload: GeminiGenerateContentPayload): Pipeline => compose('geminiGenerateContentCountTokens', [ - emitGeminiTokenCount, + emitGeminiGenerateContentTokenCount, // A measurement goes through settlement like every other run. It provides an empty // billed set because nothing here is billable today, not because the operation is // exempt — an upstream that began charging for it would provide a non-empty one and @@ -225,9 +225,9 @@ export const geminiGenerateContentCountTokensPipeline = (payload: GeminiGenerate owns: [], }), materializeAttempt('request.chat.geminiGenerateContent'), - stripUnsupportedPartFieldsFromGemini, - stripUnsupportedToolsFromGemini, - stripSafetySettingsFromGemini, - measureGeminiAsMessages, + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, + stripSafetySettingsFromGeminiGenerateContent, + measureGeminiGenerateContentAsAnthropicMessages, ...anthropicMessagesCountTokensWire, ]); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts index 8a17592b2b..756baadc10 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts @@ -3,7 +3,7 @@ /** What a Gemini client is sent when a turn produced no content. `error.status` is the * Google-RPC name and clients read it, so this shape is the protocol's rather than the * gateway's — an OpenAI-shaped envelope would leave that field undefined. */ -export const renderGeminiError = (status: number, message: string): Record => ({ +export const renderGeminiGenerateContentError = (status: number, message: string): Record => ({ error: { code: status, message, status: geminiGenerateContentStatusForHttpStatus(status) }, }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts index 115a5556d0..bc81900573 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts @@ -7,13 +7,13 @@ // it over, and turn what the run answered with into a response. Everything between is stages. import { geminiGenerateContentCountTokensPipeline } from './count-tokens.ts'; -import { renderGeminiError } from './errors.ts'; +import { renderGeminiGenerateContentError } from './errors.ts'; import { geminiGenerateContentServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; -import { createNonResponsesSourceStore } from '../openai-responses/items/store.ts'; +import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; import { move } from '@floway-dev/pipeline'; import type { GeminiGenerateContentContent, GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; @@ -30,9 +30,9 @@ interface GeminiGenerateContentModelAction { // SDKs send it). /** A refusal about the route itself. It is answered before a model is resolved, so there is no * attempt to record and nothing for a run to hold beyond the refusal. */ -const unknownAction = (message: string): Response => Response.json(renderGeminiError(404, message), { status: 404 }); +const unknownAction = (message: string): Response => Response.json(renderGeminiGenerateContentError(404, message), { status: 404 }); -const parseGeminiModelAction = (modelAction: string | undefined): GeminiGenerateContentModelAction | Response => { +const parseGeminiGenerateContentModelAction = (modelAction: string | undefined): GeminiGenerateContentModelAction | Response => { if (!modelAction) return unknownAction('Missing Gemini model action.'); const separator = modelAction.lastIndexOf(':'); if (separator <= 0 || separator === modelAction.length - 1) return unknownAction(`Unknown Gemini model action: ${modelAction}`); @@ -42,7 +42,7 @@ const parseGeminiModelAction = (modelAction: string | undefined): GeminiGenerate // `:countTokens` can carry either `contents` directly or a nested // `generateContentRequest` envelope (Google's SDK shape). Normalize both to a // single `GeminiGenerateContentPayload` for the rest of the chain. -const parseGeminiCountTokensPayload = (body: unknown): GeminiGenerateContentPayload => { +const parseGeminiGenerateContentCountTokensPayload = (body: unknown): GeminiGenerateContentPayload => { const shape = (body ?? {}) as { contents?: GeminiGenerateContentContent[]; generateContentRequest?: GeminiGenerateContentPayload }; return shape.generateContentRequest ?? { contents: shape.contents }; }; @@ -52,11 +52,11 @@ const parseGeminiCountTokensPayload = (body: unknown): GeminiGenerateContentPayl // means the sub-handlers see a validated `(model, action)` pair and never // need to re-emit "Unknown Gemini model action" on already-validated input. export const geminiGenerateContentHttp = async (c: AuthedContext): Promise => { - const parsed = parseGeminiModelAction(c.req.param('modelAction')); + const parsed = parseGeminiGenerateContentModelAction(c.req.param('modelAction')); if (parsed instanceof Response) return parsed; - if (parsed.action === 'countTokens') return await runGeminiCountTokens(c, parsed.model); + if (parsed.action === 'countTokens') return await runGeminiGenerateContentCountTokens(c, parsed.model); if (parsed.action === 'generateContent' || parsed.action === 'streamGenerateContent') { - return await runGeminiGenerate(c, parsed.model, parsed.action === 'streamGenerateContent'); + return await runGeminiGenerateContentGenerate(c, parsed.model, parsed.action === 'streamGenerateContent'); } return unknownAction(`Unknown Gemini model action: ${parsed.action}`); }; @@ -77,10 +77,10 @@ const readRequest = (bytes: Uint8Array, project: (body: unknown) => T): { typ const refuse = (c: AuthedContext, ingress: Ingress, message: string): Response => { const refused = openPrologue(c, ingress, { wantsStream: false }); refused.gateway.dump?.error('gateway'); - return finalizeGatewayResponse(refused.gateway, Response.json(renderGeminiError(400, message), { status: 400 })); + return finalizeGatewayResponse(refused.gateway, Response.json(renderGeminiGenerateContentError(400, message), { status: 400 })); }; -const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: boolean): Promise => { +const runGeminiGenerateContentGenerate = async (c: AuthedContext, model: string, wantsStream: boolean): Promise => { const ingress = await readIngress(c); const request = readRequest(ingress.body.bytes, (body): GeminiGenerateContentPayload => body as GeminiGenerateContentPayload); if (request.type === 'invalid') return refuse(c, ingress, request.message); @@ -89,7 +89,7 @@ const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: b const prologue = openChatPrologue(c, ingress, { wantsStream, model, - storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), }); return await serveThrough( @@ -98,7 +98,7 @@ const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: b geminiGenerateContentServePipeline(payload), move({ 'ingress.http.headers': prologue.headers, - 'ingress.chat.sourceProtocol': 'gemini', + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', 'ingress.chat.geminiGenerateContent.wantsStream': wantsStream, 'request.chat.geminiGenerateContent': payload, // Gemini carries the model in the path rather than the body, so the id the run @@ -114,16 +114,16 @@ const runGeminiGenerate = async (c: AuthedContext, model: string, wantsStream: b ); }; -const runGeminiCountTokens = async (c: AuthedContext, model: string): Promise => { +const runGeminiGenerateContentCountTokens = async (c: AuthedContext, model: string): Promise => { const ingress = await readIngress(c); - const request = readRequest(ingress.body.bytes, parseGeminiCountTokensPayload); + const request = readRequest(ingress.body.bytes, parseGeminiGenerateContentCountTokensPayload); if (request.type === 'invalid') return refuse(c, ingress, request.message); const { payload } = request; const prologue = openChatPrologue(c, ingress, { wantsStream: false, model, - storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), }); return await serveThrough( @@ -132,7 +132,7 @@ const runGeminiCountTokens = async (c: AuthedContext, model: string): Promise { - delete ctx.payload.safetySettings; - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields.ts deleted file mode 100644 index 0903d58900..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { GeminiGenerateContentInterceptor } from './types.ts'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentPart } from '@floway-dev/protocols/gemini-generate-content'; - -/** - * Gemini generateContent file/code parts have no current equivalent in the upstream target - * graph. Drop them at source so every Gemini generateContent route target sees translatable - * parts. - */ -const stripPartFields = (parts: GeminiGenerateContentPart[]): GeminiGenerateContentPart[] => - parts.filter(part => { - delete part.fileData; - delete part.executableCode; - delete part.codeExecutionResult; - return Object.keys(part).length > 0; - }); - -export const stripUnsupportedPartFieldsFromPayload = (payload: GeminiGenerateContentPayload): void => { - payload.contents?.forEach(content => { - content.parts = stripPartFields(content.parts); - }); - if (payload.systemInstruction) { - payload.systemInstruction.parts = stripPartFields(payload.systemInstruction.parts); - } -}; - -export const stripUnsupportedPartFields: GeminiGenerateContentInterceptor = (ctx, _gatewayCtx, run) => { - stripUnsupportedPartFieldsFromPayload(ctx.payload); - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/types.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/types.ts deleted file mode 100644 index e81d2e52f0..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/types.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import type { ExecuteResult, GeminiGenerateContentInvocation, PlainResult } from '@floway-dev/provider'; - -export type GeminiGenerateContentInterceptor = Interceptor< - GeminiGenerateContentInvocation, - GatewayCtx, - ExecuteResult> ->; - -// countTokens is a one-shot, non-streaming HTTP exchange — the terminal -// returns a `PlainResult` carrying the reshaped Gemini generateContent envelope, not an event -// stream. The interceptor chain still runs against a `GeminiGenerateContentInvocation` so -// payload-shaped reads stay symmetric with the generate path. Interceptors -// registered here MUST be pure header/payload mutators; post-`run()` result -// inspection is not portable to this result type. -export type GeminiGenerateContentCountTokensInterceptor = Interceptor< - GeminiGenerateContentInvocation, - GatewayCtx, - PlainResult ->; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index 10dbaabbef..f1f19fd9fe 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -1,18 +1,18 @@ // Gemini as a pipeline — the same chain as Chat Completions, for the one source-only family. // -// emitGemini the edge: writes the answer in the shape the client asked for +// emitGeminiGenerateContent the edge: writes the answer in the shape the client asked for // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record -// the strippers, then requireGeminiTerminal +// the strippers, then requireGeminiGenerateContentTerminal // dialChatWire the ending: picks this candidate's wire and hands into it // // Gemini is source-only. Nothing translates *into* it, so it contributes one pipeline rather // than two — a source role and no target role, and no other family's chain ever hands up // `response.chat.geminiGenerateContent`. // -// It is also the one family with no wire of its own: the provider surface has no `callGemini`, +// It is also the one family with no wire of its own: the provider surface has no `callGeminiGenerateContent`, // so every candidate is reached through a translation and all three wires are translated ones // — a handoff into Chat Completions, into Messages or into Responses, followed by that // protocol's own wire. Which one a candidate is dialled on is the picker's answer for that @@ -22,7 +22,7 @@ // spoke: the target protocol's own wire meters its own frames on their way into the // translation, so a figure the upstream stated is billed as stated. Where the *turn* ends is // a different question and is stated on the Gemini frames the client is handed, which is what -// `requireGeminiTerminal` is for — a Chat Completions stream that closed cleanly without ever +// `requireGeminiGenerateContentTerminal` is for — a Chat Completions stream that closed cleanly without ever // reporting a finish reason translates into candidates that never finish. // // `:countTokens` is not one of the three: it is a second operation over this protocol rather @@ -47,9 +47,9 @@ // maps to `INTERNAL`. That part is a departure from the replaced surface, which raised such a // status to 500. -import { wrapGeminiAffinityEgress } from './affinity/egress.ts'; -import { analyzeGeminiAffinity } from './affinity/ingress.ts'; -import { renderGeminiError } from './errors.ts'; +import { wrapGeminiGenerateContentAffinityEgress } from './affinity/egress.ts'; +import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; +import { renderGeminiGenerateContentError } from './errors.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; @@ -60,10 +60,10 @@ import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.t import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { - stripSafetySettingsFromGemini, - stripUnsupportedPartFieldsFromGemini, - stripUnsupportedToolsFromGemini, - suppressThoughtPartsFromGemini, + stripSafetySettingsFromGeminiGenerateContent, + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, + suppressThoughtPartsFromGeminiGenerateContent, } from '../interceptors.ts'; import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; @@ -73,19 +73,19 @@ import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type Cha import { compose, defineStage, move, transform, type Pipeline } from '@floway-dev/pipeline'; import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { - collectGeminiProtocolEventsToResult, + collectGeminiGenerateContentProtocolEventsToResult, geminiGenerateContentProtocolFrameToSSEFrame, - GEMINI_MISSING_TERMINAL_MESSAGE, - isGeminiTerminalEvent, + GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, + isGeminiGenerateContentTerminalEvent, type GeminiGenerateContentPayload, type GeminiGenerateContentStreamEvent, } from '@floway-dev/protocols/gemini-generate-content'; import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; -import { translateGeminiViaChatCompletions, translateGeminiViaMessages, translateGeminiViaResponses } from '@floway-dev/translate'; +import { translateGeminiGenerateContentViaOpenAIChatCompletions, translateGeminiGenerateContentViaAnthropicMessages, translateGeminiGenerateContentViaOpenAIResponses } from '@floway-dev/translate'; /** `:generateContent` has no wire of its own, so the whole preference list is translated: * Chat Completions first, then Messages, then Responses. */ -export const geminiGenerateContentTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); +export const geminiGenerateContentTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); /** What this family adds to the chat space. */ export interface GeminiGenerateContentFacts extends ChatFacts { @@ -108,14 +108,14 @@ type C = { [P in K]: GeminiGenerateC * Collecting is therefore the edge's own work and not a second reading of the upstream: the * same frames that would have gone out are folded here instead. */ -const emitGemini = defineStage< +const emitGeminiGenerateContent = defineStage< C<'ingress.chat.geminiGenerateContent.wantsStream'>, C<'ingress.chat.geminiGenerateContent.wantsStream'>, C<'ingress.chat.geminiGenerateContent.wantsStream' | 'response.chat.geminiGenerateContent' | 'response.http.headers'>, C<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ - name: 'emitGemini', + name: 'emitGeminiGenerateContent', through: { request: { needs: ['ingress.chat.geminiGenerateContent.wantsStream'], @@ -141,7 +141,7 @@ const emitGemini = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.chat.geminiGenerateContent.rendered': move(renderFailure(answer, () => renderGeminiError(answer.status, answer.message))), + 'response.chat.geminiGenerateContent.rendered': move(renderFailure(answer, () => renderGeminiGenerateContentError(answer.status, answer.message))), 'response.http.status': answer.status, }; } @@ -159,7 +159,7 @@ const emitGemini = defineStage< // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. const frames = recordFrames( - wrapGeminiAffinityEgress( + wrapGeminiGenerateContentAffinityEgress( answer.frames as AsyncIterable>, affinityEgressOptions(use.gateway), ), @@ -170,7 +170,7 @@ const emitGemini = defineStage< ...rest, 'response.http.headers': forClient, 'response.chat.geminiGenerateContent.rendered': move( - await collectGeminiProtocolEventsToResult(frames) as unknown as Record, + await collectGeminiGenerateContentProtocolEventsToResult(frames) as unknown as Record, ), 'response.http.status': 200, }; @@ -208,13 +208,13 @@ const renderSSE = (frames: AsyncIterable, Record, C<'response.chat.geminiGenerateContent'>, C<'response.chat.geminiGenerateContent'> >({ - name: 'requireGeminiTerminal', + name: 'requireGeminiGenerateContentTerminal', through: { request: { needs: [], consumes: [], provides: [] }, response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: ['response.chat.geminiGenerateContent'] }, @@ -249,9 +249,9 @@ const framesUntilTerminal = async function* ( ): AsyncGenerator> { for await (const frame of frames) { yield frame; - if (frame.type === 'done' || isGeminiTerminalEvent(frame.event)) return; + if (frame.type === 'done' || isGeminiGenerateContentTerminalEvent(frame.event)) return; } - throw new Error(GEMINI_MISSING_TERMINAL_MESSAGE); + throw new Error(GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE); }; /** This family's own reading, which every wire under it hands up. */ @@ -262,30 +262,30 @@ const STREAMED_USAGE = 'response.chat.geminiGenerateContent.streamedUsage'; const geminiGenerateContentWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { const context = { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens }; switch (target) { - case 'chat-completions': + case 'openaiChatCompletions': return compose('geminiGenerateContentViaOpenAIChatCompletions', [ handOff({ from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, - trip: async payload => await translateGeminiViaChatCompletions(payload, context), + trip: async payload => await translateGeminiGenerateContentViaOpenAIChatCompletions(payload, context), }), ...openaiChatCompletionsWire(STREAMED_USAGE), ]); - case 'messages': + case 'anthropicMessages': return compose('geminiGenerateContentViaAnthropicMessages', [ handOff({ from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, - trip: async payload => await translateGeminiViaMessages(payload, context), + trip: async payload => await translateGeminiGenerateContentViaAnthropicMessages(payload, context), }), ...anthropicMessagesWire(STREAMED_USAGE), ]); - case 'responses': + case 'openaiResponses': return compose('geminiGenerateContentViaOpenAIResponses', [ handOff({ from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, - trip: async payload => await translateGeminiViaResponses(payload, context), + trip: async payload => await translateGeminiGenerateContentViaOpenAIResponses(payload, context), }), ...openaiResponsesWire(STREAMED_USAGE), ]); @@ -297,7 +297,7 @@ const geminiGenerateContentWireFor = (target: ChatTargetApi, candidate: ModelCan * built from the request rather than being a constant. */ const narrowing = (payload: GeminiGenerateContentPayload): ChatNarrowing> => ({ canServe: candidate => geminiGenerateContentTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeGeminiAffinity(payload, gateway.affinity.codec), + affinity: async gateway => await analyzeGeminiGenerateContentAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the Gemini generateContent endpoint.`, refuse: (status, message) => ({ 'response.chat.geminiGenerateContent': { status, message }, @@ -320,7 +320,7 @@ export type GeminiGenerateContentServeExit = C< export const geminiGenerateContentServePipeline = (payload: GeminiGenerateContentPayload): Pipeline => compose('geminiGenerateContentServe', [ - emitGemini, + emitGeminiGenerateContent, writeSettlement( handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), handedUp => (handedUp as { 'response.chat.geminiGenerateContent.streamedUsage'?: unknown })['response.chat.geminiGenerateContent.streamedUsage'] !== null, @@ -331,11 +331,11 @@ export const geminiGenerateContentServePipeline = (payload: GeminiGenerateConten owns: [], }), materializeAttempt('request.chat.geminiGenerateContent'), - stripUnsupportedPartFieldsFromGemini, - stripUnsupportedToolsFromGemini, - stripSafetySettingsFromGemini, - suppressThoughtPartsFromGemini, - requireGeminiTerminal, + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, + stripSafetySettingsFromGeminiGenerateContent, + suppressThoughtPartsFromGeminiGenerateContent, + requireGeminiGenerateContentTerminal, dialChatWire({ source: 'request.chat.geminiGenerateContent', needs: ['request.chat.geminiGenerateContent', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/respond.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/respond.ts deleted file mode 100644 index 5a9949a139..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/respond.ts +++ /dev/null @@ -1,215 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapGeminiGenerateContentAffinityEgress } from './affinity/egress.ts'; -import { geminiGenerateContentStatusForHttpStatus } from './errors.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import { eventFrame, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; -import { geminiGenerateContentProtocolFrameToSSEFrame, GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, isGeminiGenerateContentErrorEvent, isGeminiGenerateContentTerminalEvent, collectGeminiGenerateContentProtocolEventsToResult } from '@floway-dev/protocols/gemini-generate-content'; -import type { GeminiGenerateContentErrorResponse, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import { type ExecuteResult, type PlainResult, type ApiErrorResult, type InternalDebugError, toInternalDebugError, decodeApiErrorBody } from '@floway-dev/provider'; - -// Renders an upstream Gemini generateContent result into the client HTTP/SSE response, in the -// Google-RPC error envelope. An error-typed result is a pre-stream failure and -// always answers as HTTP; an events result drains to one JSON body -// (non-streaming) or is proxied frame by frame (streaming). -export const respondGeminiGenerateContent = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - ctx: GatewayCtx, -): Promise => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return geminiGenerateContentApiErrorResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return geminiGenerateContentErrorResponse(result.status, result.error.message, internalDebugFields(result.error)); - } - - if (result.type === 'plain') { - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); - } - - const state = new SourceStreamState(); - const observed = observeGeminiGenerateContentFrames(result.events, state, ctx); - const frames = wrapGeminiGenerateContentAffinityEgress(observed, affinityEgressOptions(ctx)); - - if (!wantsStream) { - try { - const response = await collectGeminiGenerateContentProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return geminiGenerateContentCollectErrorResponse(error); - } - } - - forwardUpstreamHeaders(c, result.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, geminiGenerateContentSseFrames(frames, state, ctx), { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`gemini stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); -}; - -// --- error rendering: Google-RPC envelope --- - -type GeminiGenerateContentErrorDebugFields = Partial> & { target_api?: string }; - -type GeminiGenerateContentErrorStatusPayload = { - error: GeminiGenerateContentErrorResponse['error'] & GeminiGenerateContentErrorDebugFields; -}; - -const googleRpcHttpStatusCode = (status: number): number => (Number.isInteger(status) && status >= 400 && status <= 599 ? status : 500); - -const geminiGenerateContentRpcErrorPayload = (status: number, message: string, debug: GeminiGenerateContentErrorDebugFields = {}): GeminiGenerateContentErrorStatusPayload => { - const code = googleRpcHttpStatusCode(status); - return { - error: { code, message, status: geminiGenerateContentStatusForHttpStatus(code), ...debug }, - }; -}; - -const internalDebugFields = (error: InternalDebugError): GeminiGenerateContentErrorDebugFields => ({ - type: error.type, - name: error.name, - stack: error.stack, - cause: error.cause, - ...(error.target_api ? { target_api: error.target_api } : {}), -}); - -const geminiGenerateContentInternalRpcErrorPayload = (status: number, error: unknown): GeminiGenerateContentErrorStatusPayload => { - const debug = toInternalDebugError(error); - return geminiGenerateContentRpcErrorPayload(status, debug.message, internalDebugFields(debug)); -}; - -// Response builders. The count_tokens path under `http.ts` reuses them -// alongside the error renderer for its synthesized JSON envelope. -export const geminiGenerateContentRpcErrorResponse = (status: number, message: string): Response => { - const payload = geminiGenerateContentRpcErrorPayload(status, message); - return Response.json(payload, { status: payload.error.code }); -}; - -export const geminiGenerateContentInternalRpcErrorResponse = (status: number, error: unknown): Response => { - const payload = geminiGenerateContentInternalRpcErrorPayload(status, error); - return Response.json(payload, { status: payload.error.code }); -}; - -const geminiGenerateContentErrorResponse = (status: number, message: string, debug: GeminiGenerateContentErrorDebugFields = {}): Response => { - // For gateway-minted errors, a non-500 that maps to INTERNAL is coerced to 500. - const code = geminiGenerateContentStatusForHttpStatus(status) === 'INTERNAL' && status !== 500 ? 500 : status; - return Response.json({ error: { code, message, status: geminiGenerateContentStatusForHttpStatus(code), ...debug } }, { status: code }); -}; - -const geminiGenerateContentApiErrorResponse = (error: ApiErrorResult): Response => googleRpcErrorPassthroughResponse(error) ?? geminiGenerateContentErrorResponse(error.status, apiErrorMessage(error)); - -const geminiGenerateContentCollectErrorResponse = (error: unknown): Response => { - const geminiGenerateContentError = caughtGeminiGenerateContentErrorEvent(error); - return geminiGenerateContentError - ? Response.json(geminiGenerateContentError, { status: googleRpcHttpStatusCode(geminiGenerateContentError.error.code) }) - : geminiGenerateContentInternalRpcErrorResponse(502, error); -}; - -// Recognizing / extracting an upstream-shaped Gemini generateContent error from a raw body or a -// thrown cause, so a native Google error is forwarded verbatim rather than -// re-wrapped. -const parseJson = (value: string): unknown => { - try { - return JSON.parse(value); - } catch { - return undefined; - } -}; - -const isGeminiGenerateContentErrorResponse = (value: unknown): value is GeminiGenerateContentErrorResponse => { - if (!value || typeof value !== 'object' || !('error' in value)) return false; - const error = (value as { error?: unknown }).error; - if (!error || typeof error !== 'object') return false; - const payload = error as Partial; - return typeof payload.code === 'number' && typeof payload.message === 'string' && typeof payload.status === 'string'; -}; - -const googleRpcErrorPassthroughResponse = (error: ApiErrorResult): Response | null => { - const parsed = parseJson(decodeApiErrorBody(error).trim()); - if (!isGeminiGenerateContentErrorResponse(parsed)) return null; - - return new Response(error.body.slice(), { - status: googleRpcHttpStatusCode(parsed.error.code), - headers: new Headers(error.headers), - }); -}; - -const apiErrorMessage = (error: ApiErrorResult): string => { - const body = decodeApiErrorBody(error).trim(); - return body || 'Upstream Gemini generateContent request failed.'; -}; - -const caughtGeminiGenerateContentErrorEvent = (error: unknown): GeminiGenerateContentErrorResponse | null => { - if (!(error instanceof Error)) return null; - return isGeminiGenerateContentErrorResponse(error.cause) ? error.cause : null; -}; - -const geminiGenerateContentStreamErrorEvent = (error: unknown): GeminiGenerateContentStreamEvent => - (caughtGeminiGenerateContentErrorEvent(error) ?? geminiGenerateContentInternalRpcErrorPayload(500, error)) as unknown as GeminiGenerateContentStreamEvent; - -// --- frame observation --- - -const isGeminiGenerateContentTerminalFrame = (frame: ProtocolFrame): boolean => frame.type === 'done' || (frame.type === 'event' && isGeminiGenerateContentTerminalEvent(frame.event)); - -const observeGeminiGenerateContentFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - for await (const frame of frames) { - ctx.dump?.frame(frame); - const failed = frame.type === 'event' && isGeminiGenerateContentErrorEvent(frame.event); - if (failed) state.failed = true; - if (isGeminiGenerateContentTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isGeminiGenerateContentTerminalFrame(frame)) return; - } - throw new Error(GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE); -}; - -const geminiGenerateContentSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - try { - for await (const frame of frames) { - const sse = geminiGenerateContentProtocolFrameToSSEFrame(frame); - if (sse) yield sse; - } - } catch (error) { - state.failed = true; - const event = geminiGenerateContentStreamErrorEvent(error); - ctx.dump?.frame(eventFrame(event)); - yield sseFrame(JSON.stringify(event)); - } -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/serve.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/serve.ts deleted file mode 100644 index f9e7e3d7e4..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/serve.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; -import { geminiGenerateContentAttempt, geminiGenerateContentCountTokensTarget, geminiGenerateContentGenerateTarget } from './attempt.ts'; -import { renderGeminiGenerateContentFailure } from './errors.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import type { ExecuteResult, PlainResult } from '@floway-dev/provider'; - -export interface GeminiGenerateContentServeGenerateArgs { - readonly payload: GeminiGenerateContentPayload; - readonly ctx: ChatGatewayCtx; - // Per-request model id (Gemini generateContent carries it in the URL path, not the body), - // resolved by the HTTP entry and threaded through here so candidate - // enumeration and failure rendering all see the same value. - readonly model: string; - readonly headers: Headers; -} - -export interface GeminiGenerateContentServeCountTokensArgs { - readonly payload: GeminiGenerateContentPayload; - readonly ctx: ChatGatewayCtx; - readonly model: string; - readonly headers: Headers; -} - -export const geminiGenerateContentServe = { - generate: async (args: GeminiGenerateContentServeGenerateArgs): Promise>> => { - const { payload, ctx, model, headers } = args; - const affinity = await analyzeGeminiGenerateContentAffinity(payload, ctx.affinity.codec); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = enumerated.filter(c => geminiGenerateContentGenerateTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderGeminiGenerateContentFailure(selection, 'generate'); - if (selection.candidates.length === 0) return renderGeminiGenerateContentFailure(noViableCandidateFailure(sawModel, model, failedUpstreams), 'generate'); - - // Gemini generateContent carries the requested model in its URL, so affinity preparation - // owns each candidate payload while dispatch uses the candidate's canonical model. - return await iterateCandidates( - selection.candidates, - 'geminiGenerateContentServe.generate', - ctx, - 'chat', - async candidate => { - const result = await geminiGenerateContentAttempt.generate({ payload: selection.payloadFor(candidate), ctx, candidate, headers }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - }, - - countTokens: async (args: GeminiGenerateContentServeCountTokensArgs): Promise> | PlainResult> => { - const { payload, ctx, model, headers } = args; - const affinity = await analyzeGeminiGenerateContentAffinity(payload, ctx.affinity.codec); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = enumerated.filter(c => geminiGenerateContentCountTokensTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderGeminiGenerateContentFailure(selection, 'countTokens'); - if (selection.candidates.length === 0) return renderGeminiGenerateContentFailure(noViableCandidateFailure(sawModel, model, failedUpstreams), 'countTokens'); - - return await iterateCandidates( - selection.candidates, - 'geminiGenerateContentServe.countTokens', - ctx, - 'chat', - candidate => geminiGenerateContentAttempt.countTokens({ payload: selection.payloadFor(candidate), ctx, candidate, headers }), - ); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index 3ac3386f60..f1393714a0 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -33,7 +33,7 @@ import type { import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { GeminiGenerateContentContent, GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent, GeminiGenerateContentToolGroup } from '@floway-dev/protocols/gemini-generate-content'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -import type { CanonicalResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; /** Removing a field inside a value is the same move as removing a fact, and a value that * carried none of them comes back by identity so the rule costs nothing where it does not @@ -161,7 +161,7 @@ const attemptIdentity = (attempt: AttemptSelector): string => `${attempt.upstrea * are authoritative when flags overlap, and the last step affects only a system message * that appears after the leading run — which is what "mid-conversation" means. The fold * itself is `roleRewriter`; what is here is the walk over this protocol's messages. */ -export const applyRoleCompatibilityToChatCompletions = defineStage< +export const applyRoleCompatibilityToOpenAIChatCompletions = defineStage< Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, Chat<'request.chat.openaiChatCompletions'>, Chat<'response.chat.openaiChatCompletions'>, @@ -188,7 +188,7 @@ export const applyRoleCompatibilityToChatCompletions = defineStage< const rewrite = rolesFor(facts['route.attempt'].flags); if (rewrite === null) return facts; // the free do-nothing path const payload = facts['request.chat.openaiChatCompletions']; - const messages = rewriteChatCompletionsRoles(payload.messages, rewrite); + const messages = rewriteOpenAIChatCompletionsRoles(payload.messages, rewrite); // Written conditionally the whole way down, so a conversation this does not touch // comes back by identity and the layer costs what the rewrite actually changed. return messages === payload.messages @@ -219,7 +219,7 @@ const rolesFor = (flags: readonly string[]): RoleRewrite | null => { type OpenAIChatCompletionsMessages = OpenAIChatCompletionsPayload['messages']; -const rewriteChatCompletionsRoles = (messages: OpenAIChatCompletionsMessages, rewrite: RoleRewrite): OpenAIChatCompletionsMessages => { +const rewriteOpenAIChatCompletionsRoles = (messages: OpenAIChatCompletionsMessages, rewrite: RoleRewrite): OpenAIChatCompletionsMessages => { const nextRole = roleRewriter(rewrite); return mapKeepingIdentity(messages, message => { const role = nextRole(message.role); @@ -255,7 +255,7 @@ type RewrittenRole = 'system' | 'developer' | 'user'; /** The reasoning sentinel a forced tool choice needs. Gated by its flag, and the sentinel * is the gateway's canonical form — putting it on the wire in a vendor's shape is the * vendor normalizer's job, which is why this runs above them. */ -export const disableReasoningOnForcedToolChoiceForChatCompletions = defineStage< +export const disableReasoningOnForcedToolChoiceForOpenAIChatCompletions = defineStage< Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, Chat<'request.chat.openaiChatCompletions'>, Chat<'response.chat.openaiChatCompletions'>, @@ -293,7 +293,7 @@ const isForcedToolChoice = (choice: OpenAIChatCompletionsPayload['tool_choice']) /** Drops a field the upstream would reject as an unknown argument. It runs above the vendor * normalizers so each of them sees the already-stripped canonical payload. */ -export const stripPromptCacheKeyForChatCompletions = defineStage< +export const stripPromptCacheKeyForOpenAIChatCompletions = defineStage< Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, Chat<'request.chat.openaiChatCompletions'>, Chat<'response.chat.openaiChatCompletions'>, @@ -341,7 +341,7 @@ export const stripPromptCacheKeyForChatCompletions = defineStage< * * Reference: https://platform.openai.com/docs/api-reference/chat/create */ -export const includeUsageStreamOptionsForChatCompletions = defineStage< +export const includeUsageStreamOptionsForOpenAIChatCompletions = defineStage< Chat<'request.chat.openaiChatCompletions'>, Chat<'request.chat.openaiChatCompletions'>, Chat<'response.chat.openaiChatCompletions'>, @@ -390,7 +390,7 @@ export const includeUsageStreamOptionsForChatCompletions = defineStage< * It runs above the vendor normalizers, so on the way back it sees a usage block whose cache * fields already carry OpenAI's names. */ -export const normalizeUsageForChatCompletions = defineStage< +export const normalizeUsageForOpenAIChatCompletions = defineStage< Record, Record, Chat<'response.chat.openaiChatCompletions'>, @@ -459,7 +459,7 @@ const withUsageOnItsOwnCarrier = async function* ( * It runs below the vendor normalizers, so on the way back it reads usage whose cache fields * already carry OpenAI's names. */ -export const normalizeExclusiveCachedTokensForChatCompletions = defineStage< +export const normalizeExclusiveCachedTokensForOpenAIChatCompletions = defineStage< Chat<'route.attempt'>, Chat<'route.attempt'>, Chat<'response.chat.openaiChatCompletions'>, @@ -493,7 +493,7 @@ export const normalizeExclusiveCachedTokensForChatCompletions = defineStage< const identity = attemptIdentity(attempt); const answer = answerWithFrames( facts['response.chat.openaiChatCompletions'], - frames => rewritingEvents(frames, chunk => foldChatCompletionsUsage(chunk, declaredExclusive, identity)), + frames => rewritingEvents(frames, chunk => foldOpenAIChatCompletionsUsage(chunk, declaredExclusive, identity)), ); return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; }, @@ -503,21 +503,21 @@ export const normalizeExclusiveCachedTokensForChatCompletions = defineStage< /** Chat Completions carries usage on the chunk's own root and names the buckets * `prompt_tokens` and `prompt_tokens_details.{cached_tokens, cache_creation_input_tokens}`. */ -const CHAT_COMPLETIONS_CACHE_BUCKETS: CacheBucketNames = { +const OPENAI_CHAT_COMPLETIONS_CACHE_BUCKETS: CacheBucketNames = { input: 'prompt_tokens', output: 'completion_tokens', details: 'prompt_tokens_details', cacheWrite: ['cache_creation_input_tokens', 'cache_write_tokens'], }; -const foldChatCompletionsUsage = ( +const foldOpenAIChatCompletionsUsage = ( chunk: OpenAIChatCompletionsStreamEvent, declaredExclusive: boolean, identity: string, ): OpenAIChatCompletionsStreamEvent => { const usage = asJsonObject(chunk.usage); if (usage === null) return chunk; - const folded = withCacheBucketsFolded(usage, CHAT_COMPLETIONS_CACHE_BUCKETS, declaredExclusive, identity); + const folded = withCacheBucketsFolded(usage, OPENAI_CHAT_COMPLETIONS_CACHE_BUCKETS, declaredExclusive, identity); return folded === usage ? chunk : { ...chunk, usage: folded as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; }; @@ -548,7 +548,7 @@ const foldChatCompletionsUsage = ( * - https://api-docs.deepseek.com/guides/kv_cache * - https://api-docs.deepseek.com/quick_start/agent_integrations/oh_my_pi */ -export const vendorDeepSeekNormalizeForChatCompletions = defineStage< +export const vendorDeepSeekNormalizeForOpenAIChatCompletions = defineStage< Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, Chat<'request.chat.openaiChatCompletions'>, Chat<'response.chat.openaiChatCompletions'>, @@ -664,7 +664,7 @@ const deepSeekInboundUsage = (chunk: OpenAIChatCompletionsStreamEvent): OpenAICh * canonical sentinel. Its response shape matches OpenAI for the fields the gateway reads, so * there is nothing to do on the way back. * https://www.alibabacloud.com/help/en/model-studio/deep-thinking */ -export const vendorQwenNormalizeForChatCompletions = defineStage< +export const vendorQwenNormalizeForOpenAIChatCompletions = defineStage< Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, Chat<'request.chat.openaiChatCompletions'>, Chat<'response.chat.openaiChatCompletions'>, @@ -702,7 +702,7 @@ export const vendorQwenNormalizeForChatCompletions = defineStage< * rules above this one read OpenAI's `prompt_tokens_details.cached_tokens`, so it is put * there. Kimi accepts the canonical request shape, so there is nothing to do on the way out. * https://platform.kimi.com/docs/api/chat */ -export const vendorKimiNormalizeForChatCompletions = defineStage< +export const vendorKimiNormalizeForOpenAIChatCompletions = defineStage< Chat<'route.attempt'>, Chat<'route.attempt'>, Chat<'response.chat.openaiChatCompletions'>, @@ -772,7 +772,7 @@ type OpenAIChatCompletionsDeltaWithDeepSeekReasoning = * the same flag as the other protocols' fold but is not that fold: there is no leading run to * cross here, and no developer role to trade with. */ -export const applyRoleCompatibilityToMessages = defineStage< +export const applyRoleCompatibilityToAnthropicMessages = defineStage< Chat<'request.chat.anthropicMessages' | 'route.attempt'>, Chat<'request.chat.anthropicMessages'>, Chat<'response.chat.anthropicMessages'>, @@ -808,7 +808,7 @@ export const applyRoleCompatibilityToMessages = defineStage< /** The reasoning sentinel a forced tool choice needs, in the shape this protocol has * natively. `tool` and `any` are the forced choices; `auto` and `none` leave the model free * to reason. */ -export const disableReasoningOnForcedToolChoiceForMessages = defineStage< +export const disableReasoningOnForcedToolChoiceForAnthropicMessages = defineStage< Chat<'request.chat.anthropicMessages' | 'route.attempt'>, Chat<'request.chat.anthropicMessages'>, Chat<'response.chat.anthropicMessages'>, @@ -836,7 +836,7 @@ export const disableReasoningOnForcedToolChoiceForMessages = defineStage< const payload = facts['request.chat.anthropicMessages']; const choice = payload.tool_choice?.type; if (choice !== 'tool' && choice !== 'any') return facts; - return { ...facts, 'request.chat.anthropicMessages': move(withMessagesReasoningDisabled(payload)) }; + return { ...facts, 'request.chat.anthropicMessages': move(withAnthropicMessagesReasoningDisabled(payload)) }; }, })), }); @@ -845,7 +845,7 @@ export const disableReasoningOnForcedToolChoiceForMessages = defineStage< * with structured output on these upstreams, and it is thinking it does not compose with, so * `output_config.format` has to survive. An `output_config` that held nothing but the effort * is nothing at all once the effort is gone, so it goes rather than riding on empty. */ -const withMessagesReasoningDisabled = (payload: AnthropicMessagesPayload): AnthropicMessagesPayload => { +const withAnthropicMessagesReasoningDisabled = (payload: AnthropicMessagesPayload): AnthropicMessagesPayload => { const { output_config, ...rest } = payload; const next: AnthropicMessagesPayload = { ...rest, thinking: { type: 'disabled' as const } }; if (output_config !== undefined) { @@ -873,7 +873,7 @@ const withMessagesReasoningDisabled = (payload: AnthropicMessagesPayload): Anthr * does with it is not documented anywhere primary — the observable part is that the CLI adds * *more* of these fields on its first-party path, so it reads as a client-attribution signal. */ -export const stripBillingAttributionFromMessages = defineStage< +export const stripBillingAttributionFromAnthropicMessages = defineStage< Chat<'request.chat.anthropicMessages' | 'route.attempt'>, Chat<'request.chat.anthropicMessages'>, Chat<'response.chat.anthropicMessages'>, @@ -951,7 +951,7 @@ const scrubbedSystemPrompt = (system: AnthropicMessagesPayload['system']): Anthr /** Gemini file and code parts have no equivalent anywhere the translations reach, so they go * at source and every target sees translatable parts. A part left holding nothing goes with * them: it is not a part any more. */ -export const stripUnsupportedPartFieldsFromGemini = defineStage< +export const stripUnsupportedPartFieldsFromGeminiGenerateContent = defineStage< Chat<'request.chat.geminiGenerateContent'>, Chat<'request.chat.geminiGenerateContent'>, Chat<'response.chat.geminiGenerateContent'>, @@ -1014,7 +1014,7 @@ const stripContentParts = (content: GeminiGenerateContentContent): GeminiGenerat * capabilities go — and a group left declaring no function goes with them, because a target * emitter offered an empty group would be offered a tool that does nothing. `tools` itself * goes when no group survived: an empty tool list is a different request from no tools. */ -export const stripUnsupportedToolsFromGemini = defineStage< +export const stripUnsupportedToolsFromGeminiGenerateContent = defineStage< Chat<'request.chat.geminiGenerateContent'>, Chat<'request.chat.geminiGenerateContent'>, Chat<'response.chat.geminiGenerateContent'>, @@ -1075,7 +1075,7 @@ const withUnsupportedToolsStripped = (payload: GeminiGenerateContentPayload): Ge /** Gemini safety controls are source-specific and have no matching control on every target * path, so they go rather than have us pretend to enforce a policy we cannot honor * end to end. */ -export const stripSafetySettingsFromGemini = defineStage< +export const stripSafetySettingsFromGeminiGenerateContent = defineStage< Chat<'request.chat.geminiGenerateContent'>, Chat<'request.chat.geminiGenerateContent'>, Chat<'response.chat.geminiGenerateContent'>, @@ -1108,7 +1108,7 @@ export const stripSafetySettingsFromGemini = defineStage< * So it rides in this stage's own closure rather than as a declaration: a response-side * `needs` can only name what the ending provides, which is the answer and not the turn. */ -export const suppressThoughtPartsFromGemini = defineStage< +export const suppressThoughtPartsFromGeminiGenerateContent = defineStage< Chat<'request.chat.geminiGenerateContent'>, Chat<'request.chat.geminiGenerateContent'>, Chat<'response.chat.geminiGenerateContent'>, @@ -1175,11 +1175,11 @@ const withoutThoughtParts = async function* ( ...frame.event, ...(candidates === undefined ? {} : { candidates }), }; - if (hasGeminiEventPayload(event)) yield eventFrame(event); + if (hasGeminiGenerateContentEventPayload(event)) yield eventFrame(event); } }; -const hasGeminiEventPayload = (event: GeminiGenerateContentStreamEvent): boolean => { +const hasGeminiGenerateContentEventPayload = (event: GeminiGenerateContentStreamEvent): boolean => { if ('error' in event) return true; return (event.candidates?.length ?? 0) > 0 || event.usageMetadata !== undefined @@ -1193,7 +1193,7 @@ const hasGeminiEventPayload = (event: GeminiGenerateContentStreamEvent): boolean * here is the walk over this protocol's input items. Only a message item carries a role, and * an item that carries none still crosses the leading system run — a reasoning item between * two system messages is what makes the second one mid-conversation. */ -export const applyRoleCompatibilityToResponses = defineStage< +export const applyRoleCompatibilityToOpenAIResponses = defineStage< Chat<'request.chat.openaiResponses' | 'route.attempt'>, Chat<'request.chat.openaiResponses'>, Chat<'response.chat.openaiResponses'>, @@ -1219,7 +1219,7 @@ export const applyRoleCompatibilityToResponses = defineStage< if (rewrite === null) return facts; // the free do-nothing path // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to, which is the one a wire takes. - const payload = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; const nextRole = roleRewriter(rewrite); const input = mapKeepingIdentity(payload.input, (item): OpenAIResponsesInputItem => { if (item.type !== 'message') { @@ -1241,7 +1241,7 @@ export const applyRoleCompatibilityToResponses = defineStage< * rather than merged — a summary has no meaning once there is nothing to summarize — and * the sentinel is the gateway's canonical form, which is why this runs above the vendor * normalizers that put it on the wire in a vendor's shape. */ -export const disableReasoningOnForcedToolChoiceForResponses = defineStage< +export const disableReasoningOnForcedToolChoiceForOpenAIResponses = defineStage< Chat<'request.chat.openaiResponses' | 'route.attempt'>, Chat<'request.chat.openaiResponses'>, Chat<'response.chat.openaiResponses'>, @@ -1267,13 +1267,13 @@ export const disableReasoningOnForcedToolChoiceForResponses = defineStage< return facts; } const payload = facts['request.chat.openaiResponses']; - if (!isForcedResponsesToolChoice(payload.tool_choice)) return facts; + if (!isForcedOpenAIResponsesToolChoice(payload.tool_choice)) return facts; return { ...facts, 'request.chat.openaiResponses': move({ ...payload, reasoning: { effort: 'none' } }) }; }, })), }); -const isForcedResponsesToolChoice = (choice: OpenAIResponsesPayload['tool_choice']): boolean => { +const isForcedOpenAIResponsesToolChoice = (choice: OpenAIResponsesPayload['tool_choice']): boolean => { if (choice === undefined || choice === null) return false; if (typeof choice === 'string') return choice === 'required'; return true; @@ -1281,7 +1281,7 @@ const isForcedResponsesToolChoice = (choice: OpenAIResponsesPayload['tool_choice /** Drops a field the upstream would reject as an unknown argument. It runs above the vendor * normalizers so each of them sees the already-stripped canonical payload. */ -export const stripPromptCacheKeyForResponses = defineStage< +export const stripPromptCacheKeyForOpenAIResponses = defineStage< Chat<'request.chat.openaiResponses' | 'route.attempt'>, Chat<'request.chat.openaiResponses'>, Chat<'response.chat.openaiResponses'>, @@ -1323,7 +1323,7 @@ export const stripPromptCacheKeyForResponses = defineStage< * the contradictions that raise are documented at `foldsExclusiveCacheTokens` and at * `withCacheBucketsFolded`, which is the fold itself. */ -export const normalizeExclusiveCachedTokensForResponses = defineStage< +export const normalizeExclusiveCachedTokensForOpenAIResponses = defineStage< Chat<'route.attempt'>, Chat<'route.attempt'>, Chat<'response.chat.openaiResponses'>, @@ -1353,7 +1353,7 @@ export const normalizeExclusiveCachedTokensForResponses = defineStage< const identity = attemptIdentity(attempt); const answer = answerWithFrames( facts['response.chat.openaiResponses'], - frames => rewritingEvents(frames, event => foldResponsesUsage(event, declaredExclusive, identity)), + frames => rewritingEvents(frames, event => foldOpenAIResponsesUsage(event, declaredExclusive, identity)), ); return answer === null ? facts : { ...facts, 'response.chat.openaiResponses': move(answer) }; }, @@ -1363,14 +1363,14 @@ export const normalizeExclusiveCachedTokensForResponses = defineStage< /** Responses carries usage on `event.response.usage` and names the buckets `input_tokens` and * `input_tokens_details.{cached_tokens, cache_write_tokens}`. */ -const RESPONSES_CACHE_BUCKETS: CacheBucketNames = { +const OPENAI_RESPONSES_CACHE_BUCKETS: CacheBucketNames = { input: 'input_tokens', output: 'output_tokens', details: 'input_tokens_details', cacheWrite: ['cache_write_tokens'], }; -const foldResponsesUsage = ( +const foldOpenAIResponsesUsage = ( event: OpenAIResponsesStreamEvent, declaredExclusive: boolean, identity: string, @@ -1379,7 +1379,7 @@ const foldResponsesUsage = ( const response = asJsonObject(event.response); const usage = asJsonObject(response?.usage); if (response === null || usage === null) return event; - const folded = withCacheBucketsFolded(usage, RESPONSES_CACHE_BUCKETS, declaredExclusive, identity); + const folded = withCacheBucketsFolded(usage, OPENAI_RESPONSES_CACHE_BUCKETS, declaredExclusive, identity); if (folded === usage) return event; return { ...event, response: { ...response, usage: folded } as JsonObject } as unknown as OpenAIResponsesStreamEvent; }; @@ -1388,7 +1388,7 @@ const foldResponsesUsage = ( * with the canonical sentinel, so the sentinel is put into its wire form here — last among * this chain's rewrites, because a vendor normalizer has the final say on the outbound body. * https://api-docs.deepseek.com/zh-cn/guides/thinking_mode */ -export const vendorDeepSeekNormalizeForResponses = defineStage< +export const vendorDeepSeekNormalizeForOpenAIResponses = defineStage< Chat<'request.chat.openaiResponses' | 'route.attempt'>, Chat<'request.chat.openaiResponses'>, Chat<'response.chat.openaiResponses'>, @@ -1422,7 +1422,7 @@ export const vendorDeepSeekNormalizeForResponses = defineStage< /** Qwen says the same thing with a top-level `enable_thinking: false`. * https://www.alibabacloud.com/help/en/model-studio/deep-thinking */ -export const vendorQwenNormalizeForResponses = defineStage< +export const vendorQwenNormalizeForOpenAIResponses = defineStage< Chat<'request.chat.openaiResponses' | 'route.attempt'>, Chat<'request.chat.openaiResponses'>, Chat<'response.chat.openaiResponses'>, diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/attempt.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/attempt.ts deleted file mode 100644 index 4353a68088..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/attempt.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { openaiChatCompletionsInterceptors } from './interceptors/index.ts'; -import type { OpenAIChatCompletionsInvocation } from './interceptors/types.ts'; -import { billableUsageFromOpenAIChatCompletionsEvent } from './usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; -import { anthropicMessagesAttempt } from '../anthropic-messages/attempt.ts'; -import { openaiResponsesAttempt } from '../openai-responses/attempt.ts'; -import { applyRulesToUpstreamOpenAIChatCompletions } from '../shared/alias-rules.ts'; -import { createExternalImageLoader } from '../shared/external-image-loader.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerStreamResultToExecuteResult } from '../shared/provider-stream-result.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ModelCandidate, type ExecuteResult, providerModelOf } from '@floway-dev/provider'; -import { translateOpenAIChatCompletionsViaAnthropicMessages, translateOpenAIChatCompletionsViaOpenAIResponses } from '@floway-dev/translate'; - -// `/v1/chat/completions` generate prefers a native OpenAI Chat Completions target, -// then the translated Anthropic Messages path, then the translated OpenAI Responses path. -export const openaiChatCompletionsTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); - -export interface OpenAIChatCompletionsAttemptArgs { - readonly payload: OpenAIChatCompletionsPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export const openaiChatCompletionsAttempt = { - generate: async (args: OpenAIChatCompletionsAttemptArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - const targetApi = openaiChatCompletionsTarget.pick(candidate.model.endpoints); - const invocation: OpenAIChatCompletionsInvocation = { - payload, - candidate, - targetApi, - headers, - }; - return await runInterceptors(invocation, ctx, openaiChatCompletionsInterceptors, async () => { - if (targetApi === 'openaiChatCompletions') { - if (candidate.rules !== undefined) applyRulesToUpstreamOpenAIChatCompletions(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const providerResult = await candidate.provider.instance.callOpenAIChatCompletions( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildUpstreamCallOptions(candidate, ctx, invocation.headers), - ); - return await providerStreamResultToExecuteResult(providerResult, candidate, 'openaiChatCompletions', ctx, billableUsageFromOpenAIChatCompletionsEvent); - } - if (targetApi === 'anthropicMessages') { - return await traverseTranslation( - invocation.payload, - p => translateOpenAIChatCompletionsViaAnthropicMessages(p, { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - loadRemoteImage: createExternalImageLoader(ctx.abortSignal), - }), - translated => anthropicMessagesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }), - ); - } - if (targetApi === 'openaiResponses') { - return await traverseTranslation( - invocation.payload, - p => translateOpenAIChatCompletionsViaOpenAIResponses(p, { model: candidate.model.id }), - translated => openaiResponsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - ); - } - throw new Error(`openaiChatCompletionsAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts index d111f36f94..c7e91d87f5 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts @@ -14,7 +14,7 @@ import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; -import { createNonResponsesSourceStore } from '../openai-responses/items/store.ts'; +import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; import { move } from '@floway-dev/pipeline'; import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; @@ -48,7 +48,7 @@ export const openaiChatCompletionsHttp = { const prologue = openChatPrologue(c, ingress, { wantsStream, model: payload.model, - storeFactory: apiKey => createNonResponsesSourceStore(apiKey.id), + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), }); return await serveThrough( @@ -57,7 +57,7 @@ export const openaiChatCompletionsHttp = { openaiChatCompletionsServePipeline(payload), move({ 'ingress.http.headers': prologue.headers, - 'ingress.chat.sourceProtocol': 'chatCompletions', + 'ingress.chat.sourceProtocol': 'openaiChatCompletions', 'ingress.chat.openaiChatCompletions.wantsStream': wantsStream, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': payload.stream_options?.include_usage === true, 'request.chat.openaiChatCompletions': payload, diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/types.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/types.ts deleted file mode 100644 index 6d10cb8a2f..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/types.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { OpenAIChatCompletionsInvocation, ExecuteResult } from '@floway-dev/provider'; - -export type { OpenAIChatCompletionsInvocation }; - -export type OpenAIChatCompletionsInterceptor = Interceptor< - OpenAIChatCompletionsInvocation, - GatewayCtx, - ExecuteResult> ->; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index 364bd063df..d10eafe6a0 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -1,6 +1,6 @@ // Chat Completions as a pipeline — the reference chain for the chat families. // -// emitChatCompletions the edge: writes the answer in the shape the client asked for +// emitOpenAIChatCompletions the edge: writes the answer in the shape the client asked for // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate @@ -15,14 +15,14 @@ // // What sits *in* a wire rather than above the fork is the other half of the arrangement. A // rule that speaks about an upstream's Chat Completions endpoint — the role rewrite does, -// which is why its interceptor form stood down on `ctx.targetApi !== 'chat-completions'` — +// which is why its interceptor form stood down on `ctx.targetApi !== 'openaiChatCompletions'` — // belongs to the wire and not to the source chain, so a turn that leaves for another // protocol never carries it. Said by position rather than by a guard: a stage below the fork // runs only when this is the wire the fork chose. -import { wrapChatCompletionsAffinityEgress } from './affinity/egress.ts'; -import { analyzeChatCompletionsAffinity } from './affinity/ingress.ts'; -import { billableUsageFromChatCompletionsEvent } from './usage.ts'; +import { wrapOpenAIChatCompletionsAffinityEgress } from './affinity/egress.ts'; +import { analyzeOpenAIChatCompletionsAffinity } from './affinity/ingress.ts'; +import { billableUsageFromOpenAIChatCompletionsEvent } from './usage.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; @@ -37,40 +37,40 @@ import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { - applyRoleCompatibilityToChatCompletions, - disableReasoningOnForcedToolChoiceForChatCompletions, - includeUsageStreamOptionsForChatCompletions, - normalizeExclusiveCachedTokensForChatCompletions, - normalizeUsageForChatCompletions, - stripPromptCacheKeyForChatCompletions, - vendorDeepSeekNormalizeForChatCompletions, - vendorKimiNormalizeForChatCompletions, - vendorQwenNormalizeForChatCompletions, + applyRoleCompatibilityToOpenAIChatCompletions, + disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, + includeUsageStreamOptionsForOpenAIChatCompletions, + normalizeExclusiveCachedTokensForOpenAIChatCompletions, + normalizeUsageForOpenAIChatCompletions, + stripPromptCacheKeyForOpenAIChatCompletions, + vendorDeepSeekNormalizeForOpenAIChatCompletions, + vendorKimiNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIChatCompletions, } from '../interceptors.ts'; import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { applyRulesToUpstreamChatCompletions } from '../shared/alias-rules.ts'; +import { applyRulesToUpstreamOpenAIChatCompletions } from '../shared/alias-rules.ts'; import { createExternalImageLoader } from '../shared/external-image-loader.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage, type Use } from '@floway-dev/pipeline'; import { - CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, + OPENAI_CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, openaiChatCompletionsErrorPayloadMessage, openaiChatCompletionsProtocolFrameToSSEFrame, - collectChatCompletionsProtocolEventsToResult, + collectOpenAIChatCompletionsProtocolEventsToResult, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent, } from '@floway-dev/protocols/openai-chat-completions'; import type { BillableUsage, ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; -import { translateChatCompletionsViaMessages, translateChatCompletionsViaResponses } from '@floway-dev/translate'; +import { translateOpenAIChatCompletionsViaAnthropicMessages, translateOpenAIChatCompletionsViaOpenAIResponses } from '@floway-dev/translate'; /** `/v1/chat/completions` prefers its own wire, then the translated Messages path, then the * translated Responses path. */ -export const openaiChatCompletionsTarget = chatTargetPicker(['chat-completions', 'messages', 'responses']); +export const openaiChatCompletionsTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); /** What this family adds to the chat space. */ export interface OpenAIChatCompletionsFacts extends ChatFacts { @@ -92,7 +92,7 @@ type C = { [P in K]: OpenAIChatCompl * Collecting is therefore the edge's own work and not a second reading of the upstream: the * same frames that would have gone out are folded here instead. */ -const emitChatCompletions = defineStage< +const emitOpenAIChatCompletions = defineStage< C<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, C<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, C<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk' @@ -100,7 +100,7 @@ const emitChatCompletions = defineStage< C<'response.chat.openaiChatCompletions.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ - name: 'emitChatCompletions', + name: 'emitOpenAIChatCompletions', through: { request: { needs: ['ingress.chat.openaiChatCompletions.wantsStream', 'ingress.chat.openaiChatCompletions.wantsUsageChunk'], @@ -147,7 +147,7 @@ const emitChatCompletions = defineStage< // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. const frames = recordFrames( - wrapChatCompletionsAffinityEgress( + wrapOpenAIChatCompletionsAffinityEgress( answer.frames as AsyncIterable>, affinityEgressOptions(use.gateway), ), @@ -158,7 +158,7 @@ const emitChatCompletions = defineStage< ...rest, 'response.http.headers': forClient, 'response.chat.openaiChatCompletions.rendered': move( - await collectChatCompletionsProtocolEventsToResult(frames) as unknown as Record, + await collectOpenAIChatCompletionsProtocolEventsToResult(frames) as unknown as Record, ), 'response.http.status': 200, }; @@ -197,12 +197,12 @@ const renderSSE = ( * spoke, and where that reading lands is the *source* family's own key rather than this * protocol's — so it is built with it rather than naming one of its own. */ -const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< +const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage< C<'request.chat.openaiChatCompletions' | 'route.attempt' | 'ingress.http.headers'>, C<'response.chat.openaiChatCompletions' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ - name: 'callChatCompletionsUpstream', + name: 'callOpenAIChatCompletionsUpstream', return: { provides: [ 'response.chat.openaiChatCompletions', @@ -220,11 +220,11 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< // What the record holds by now: the payload affinity materialized for this candidate, // as every stage between the fork and here has rewritten it — or, on a translated wire, // what the handoff put here. - const body = bodyForAttempt(facts['request.chat.openaiChatCompletions'], candidate, applyRulesToUpstreamChatCompletions); + const body = bodyForAttempt(facts['request.chat.openaiChatCompletions'], candidate, applyRulesToUpstreamOpenAIChatCompletions); let result; try { - result = await candidate.provider.instance.callChatCompletions( + result = await candidate.provider.instance.callOpenAIChatCompletions( providerModelOf(candidate), body, use.gateway.abortSignal, @@ -272,7 +272,7 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< // This candidate answered, so it is the one a follow-up turn carrying our own state // must come back to. use.selectAffinity(candidate); - const metered = meterChatCompletions(result.events, identity, use.gateway.attempt); + const metered = meterOpenAIChatCompletions(result.events, identity, use.gateway.attempt); return move({ ...facts, 'response.chat.openaiChatCompletions': { kind: 'stream' as const, frames: metered.frames }, @@ -307,16 +307,16 @@ const callChatCompletionsUpstream = (streamedUsage: string) => defineStage< * and the field an upstream would reject is gone before a vendor rewrites what is left. */ export const openaiChatCompletionsWire = (streamedUsage: string): readonly Stage[] => [ - includeUsageStreamOptionsForChatCompletions, - normalizeUsageForChatCompletions, - disableReasoningOnForcedToolChoiceForChatCompletions, - applyRoleCompatibilityToChatCompletions, - stripPromptCacheKeyForChatCompletions, - normalizeExclusiveCachedTokensForChatCompletions, - vendorDeepSeekNormalizeForChatCompletions, - vendorQwenNormalizeForChatCompletions, - vendorKimiNormalizeForChatCompletions, - callChatCompletionsUpstream(streamedUsage), + includeUsageStreamOptionsForOpenAIChatCompletions, + normalizeUsageForOpenAIChatCompletions, + disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, + applyRoleCompatibilityToOpenAIChatCompletions, + stripPromptCacheKeyForOpenAIChatCompletions, + normalizeExclusiveCachedTokensForOpenAIChatCompletions, + vendorDeepSeekNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIChatCompletions, + vendorKimiNormalizeForOpenAIChatCompletions, + callOpenAIChatCompletionsUpstream(streamedUsage), ]; /** This family's own reading, which every wire under it hands up. */ @@ -326,14 +326,14 @@ const STREAMED_USAGE = 'response.chat.openaiChatCompletions.streamedUsage'; * translated one is a handoff and then the target protocol's own wire. */ const openaiChatCompletionsWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { switch (target) { - case 'chat-completions': + case 'openaiChatCompletions': return compose('openaiChatCompletionsNative', openaiChatCompletionsWire(STREAMED_USAGE)); - case 'messages': + case 'anthropicMessages': return compose('openaiChatCompletionsViaAnthropicMessages', [ handOff({ from: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, - trip: async payload => await translateChatCompletionsViaMessages(payload, { + trip: async payload => await translateOpenAIChatCompletionsViaAnthropicMessages(payload, { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), @@ -341,12 +341,12 @@ const openaiChatCompletionsWireFor = (target: ChatTargetApi, candidate: ModelCan }), ...anthropicMessagesWire(STREAMED_USAGE), ]); - case 'responses': + case 'openaiResponses': return compose('openaiChatCompletionsViaOpenAIResponses', [ handOff({ from: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, - trip: async payload => await translateChatCompletionsViaResponses(payload, { model: candidate.model.id }), + trip: async payload => await translateOpenAIChatCompletionsViaOpenAIResponses(payload, { model: candidate.model.id }), }), ...openaiResponsesWire(STREAMED_USAGE), ]); @@ -356,7 +356,7 @@ const openaiChatCompletionsWireFor = (target: ChatTargetApi, candidate: ModelCan /** Reads the upstream's own usage off its own events as they pass, so the reading costs one * pass and the client's stream is what drives it. Only a report carrying real counts * replaces the running figure, so a trailing empty usage frame cannot wipe a good one. */ -const meterChatCompletions = ( +const meterOpenAIChatCompletions = ( source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, @@ -372,11 +372,11 @@ const meterChatCompletions = ( for await (const frame of source) { // Time to first token is measured where the token is, which is the only place that // knows a frame carries generated content rather than the envelope around it. - if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'chat-completions')) { + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'openaiChatCompletions')) { attempt.firstOutputTokenAt = performance.now(); } if (frame.type === 'event') { - const usage = billableUsageFromChatCompletionsEvent(frame.event); + const usage = billableUsageFromOpenAIChatCompletionsEvent(frame.event); if (usage !== null) reported = usage; } yield frame; @@ -386,7 +386,7 @@ const meterChatCompletions = ( } // A stream that ran out without saying it ended is not a turn that finished. Serving // what arrived would present a truncated answer as a whole one. - throw new Error(CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE); + throw new Error(OPENAI_CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE); } finally { // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are @@ -418,7 +418,7 @@ const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModel * built from the request rather than being a constant. */ const narrowing = (payload: OpenAIChatCompletionsPayload): ChatNarrowing> => ({ canServe: candidate => openaiChatCompletionsTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeChatCompletionsAffinity(payload, gateway.affinity.codec), + affinity: async gateway => await analyzeOpenAIChatCompletionsAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /chat/completions endpoint.`, refuse: (status, message, reason) => ({ 'response.chat.openaiChatCompletions.streamedUsage': null, @@ -452,7 +452,7 @@ export type OpenAIChatCompletionsServeExit = C< export const openaiChatCompletionsServePipeline = (payload: OpenAIChatCompletionsPayload): Pipeline => compose('openaiChatCompletionsServe', [ - emitChatCompletions, + emitOpenAIChatCompletions, writeSettlement( handedUp => isFailure((handedUp as { 'response.chat.openaiChatCompletions'?: unknown })['response.chat.openaiChatCompletions']), handedUp => (handedUp as { 'response.chat.openaiChatCompletions.streamedUsage'?: unknown })['response.chat.openaiChatCompletions.streamedUsage'] !== null, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts b/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts index 49add24b87..c62fdce8a6 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts @@ -1,12 +1,12 @@ -import { wrapResponsesAffinityEgress } from './affinity/egress.ts'; -import { wrapResponsesClientOutput, wrapResponsesObservedOutput } from './items/output.ts'; -import { createResponsesResponseId } from './response-id.ts'; +import { wrapOpenAIResponsesAffinityEgress } from './affinity/egress.ts'; +import { wrapOpenAIResponsesClientOutput, wrapOpenAIResponsesObservedOutput } from './items/output.ts'; +import { createOpenAIResponsesResponseId } from './response-id.ts'; import { wrapResponseResourceCompletion } from './response-resource.ts'; import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { CanonicalResponsesPayload, ClientResponsesStreamEvent, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { CanonicalOpenAIResponsesPayload, ClientOpenAIResponsesStreamEvent, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; // Unix seconds, from the gateway's own request-start instant, so both resources // date a turn the same way. @@ -17,27 +17,27 @@ export const openaiResponsesCreatedAt = (ctx: GatewayCtx): number => Math.floor( // complete emitted item under its exact ID and applies one generated response // ID to the downstream stream and snapshot. Every native Responses turn goes // through this half, whichever resource it answers with. -export const wrapResponsesStatefulOutput = ( +export const wrapOpenAIResponsesStatefulOutput = ( frames: AsyncIterable>, ctx: ChatGatewayCtx, ): AsyncIterable> => { - const withObservedOutput = wrapResponsesObservedOutput(frames); - const withAffinity = wrapResponsesAffinityEgress(withObservedOutput, affinityEgressOptions(ctx)); - return wrapResponsesClientOutput(withAffinity, { + const withObservedOutput = wrapOpenAIResponsesObservedOutput(frames); + const withAffinity = wrapOpenAIResponsesAffinityEgress(withObservedOutput, affinityEgressOptions(ctx)); + return wrapOpenAIResponsesClientOutput(withAffinity, { store: ctx.store, - responseId: createResponsesResponseId(), + responseId: createOpenAIResponsesResponseId(), }); }; // The generate path's egress: the stateful half plus the response resource's // completion. `/responses/compact` answers with `CompactResource`, so it stops // at the stateful half and completes that resource itself. -export const wrapResponsesClientEgress = ( +export const wrapOpenAIResponsesClientEgress = ( frames: AsyncIterable>, ctx: ChatGatewayCtx, - request: CanonicalResponsesPayload, -): AsyncIterable> => - wrapResponseResourceCompletion(wrapResponsesStatefulOutput(frames, ctx), { + request: CanonicalOpenAIResponsesPayload, +): AsyncIterable> => + wrapResponseResourceCompletion(wrapOpenAIResponsesStatefulOutput(frames, ctx), { request, createdAt: openaiResponsesCreatedAt(ctx), stored: ctx.store.writesState, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts index 3c51b16fba..9c04f35e10 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts @@ -34,7 +34,7 @@ import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../shared/base64url-json.ts'; import { isJsonObject } from '../../../shared/json-helpers.ts'; -import type { CanonicalResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; // The two vendored constants below (SUMMARIZATION_PROMPT and SUMMARY_PREFIX) // are the compactor system prompt and the handoff prefix openai/codex ships @@ -221,7 +221,7 @@ export const buildCompactionEnvelope = (cmpId: string, summaryText: string, upst // Exported because the pipeline's compaction chain sends the same turn — one // definition is what keeps the simulated compaction identical whichever entry // asked for it. -export const summarizationTurnFor = (payload: CanonicalResponsesPayload): CanonicalResponsesPayload => { +export const summarizationTurnFor = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { // Strip compaction_trigger so the upstream sees a plain generate turn // against SUMMARIZATION_PROMPT. const historyItems = payload.input.filter(item => item.type !== 'compaction_trigger'); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index c7eef24cec..a8e0846015 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -1,6 +1,6 @@ // `/v1/responses/compact` as a pipeline. // -// emitResponsesCompaction the edge: persists what the turn emitted and completes the resource +// emitOpenAIResponsesCompaction the edge: persists what the turn emitted and completes the resource // writeSettlement above the fork, so a run bills once however many wires it tried // hydrateStoredItems the stored-items membrane, on the way in // resolveChatCandidates narrows to what can serve, in the order affinity asks for @@ -8,7 +8,7 @@ // materializeAttempt puts the payload this candidate is owed into the record // beginStoredAttempt reseeds the store's per-attempt scratchpad // expandShimCompactions a compaction this gateway wrote, back into what it stood for -// dialResponsesCompaction the ending: dials a compaction, or simulates one +// dialOpenAIResponsesCompaction the ending: dials a compaction, or simulates one // // A second **operation** over this protocol rather than another wire under // `openaiResponsesServePipeline`, which is why it is a chain of its own. Everything above the @@ -39,16 +39,16 @@ // a compaction whose caller declared a server tool is summarized without the ReAct loop wrapped // around it — the same gap the generate chain states, and for the same reason. -import { openaiResponsesCreatedAt, wrapResponsesStatefulOutput } from './client-output.ts'; -import { completeResponsesCompaction } from './compaction-resource.ts'; +import { openaiResponsesCreatedAt, wrapOpenAIResponsesStatefulOutput } from './client-output.ts'; +import { completeOpenAIResponsesCompaction } from './compaction-resource.ts'; import { syntheticEventsFromCompaction } from './items/output.ts'; import { beginStoredAttempt, - billedResponsesEntity, + billedOpenAIResponsesEntity, expandShimCompactions, hydrateStoredItems, internalErrorEnvelope, - RESPONSES_STREAMED_USAGE, + OPENAI_RESPONSES_STREAMED_USAGE, openaiResponsesNarrowing, openaiResponsesTarget, openaiResponsesWireFor, @@ -57,7 +57,7 @@ import { summarizeForCompaction, type OpenAIResponsesFacts, } from './pipeline.ts'; -import { billableUsageFromResponsesResult } from './usage.ts'; +import { billableUsageFromOpenAIResponsesResult } from './usage.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; @@ -69,13 +69,13 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import { dialChatWire, type ChatWire } from '../handoff.ts'; -import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; +import { applyRulesToUpstreamOpenAIResponses } from '../shared/alias-rules.ts'; import { materializeAttempt, resolveChatCandidates, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { - collectResponsesProtocolEventsToResult, - type CanonicalResponsesPayload, + collectOpenAIResponsesProtocolEventsToResult, + type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent, } from '@floway-dev/protocols/openai-responses'; import { providerModelOf } from '@floway-dev/provider'; @@ -99,14 +99,14 @@ type Compacted = { * the two require are different, and a compaction decorated with the twenty-odd keys a * response resource declares would answer with a shape the spec does not give it. */ -const emitResponsesCompaction = defineStage< +const emitOpenAIResponsesCompaction = defineStage< Record, Record, Compacted<'response.chat.openaiResponses'> & R<'response.http.headers' | 'response.chat.openaiResponses.streamedUsage'>, R<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.openaiResponses.streamedUsage'>, ChatServices >({ - name: 'emitResponsesCompaction', + name: 'emitOpenAIResponsesCompaction', through: { request: { needs: [], consumes: [], provides: [] }, response: { @@ -146,9 +146,9 @@ const emitResponsesCompaction = defineStage< // the tee sits above the stateful half, where they are this protocol's own and carry // the ids the resource below is assembled under. Reading is what records, and the fold // on the next line is the read. - const persisted = await collectResponsesProtocolEventsToResult( + const persisted = await collectOpenAIResponsesProtocolEventsToResult( recordFrames( - wrapResponsesStatefulOutput(answer.frames as AsyncIterable>, use.gateway), + wrapOpenAIResponsesStatefulOutput(answer.frames as AsyncIterable>, use.gateway), use.gateway.dump, ), ); @@ -156,7 +156,7 @@ const emitResponsesCompaction = defineStage< ...rest, 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move( - completeResponsesCompaction(persisted, openaiResponsesCreatedAt(use.gateway)) as unknown as Record, + completeOpenAIResponsesCompaction(persisted, openaiResponsesCreatedAt(use.gateway)) as unknown as Record, ), 'response.http.status': 200, 'response.chat.openaiResponses.streamedUsage': move( @@ -197,16 +197,16 @@ const withVerdict = (outcome: Promise | null, failed: boolean): P * the envelope up as the events it expands into — the same currency a generate wire hands up, * which is what lets the stateful half above read either without knowing which ran. */ -const callResponsesCompactUpstream = defineStage< +const callOpenAIResponsesCompactUpstream = defineStage< R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, Compacted<'response.chat.openaiResponses'> & R<'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, ChatServices >({ - name: 'callResponsesCompactUpstream', + name: 'callOpenAIResponsesCompactUpstream', return: { provides: [ 'response.chat.openaiResponses', - RESPONSES_STREAMED_USAGE, + OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers', ], @@ -217,15 +217,15 @@ const callResponsesCompactUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - const asked = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; + const asked = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; // Neither field belongs on this endpoint: `store` is a gateway-only snapshot hint the // compaction endpoint rejects, and `stream` describes how an answer would be delivered // where there is one body and no stream. - const { stream: _stream, store: _store, ...body } = bodyForAttempt(asked, candidate, applyRulesToUpstreamResponses); + const { stream: _stream, store: _store, ...body } = bodyForAttempt(asked, candidate, applyRulesToUpstreamOpenAIResponses); let result; try { - result = await candidate.provider.instance.callResponses( + result = await candidate.provider.instance.callOpenAIResponses( providerModelOf(candidate), body, 'compact', @@ -242,7 +242,7 @@ const callResponsesCompactUpstream = defineStage< return move({ ...facts, 'response.chat.openaiResponses': { status: 502, message: error instanceof Error ? error.message : String(error) }, - [RESPONSES_STREAMED_USAGE]: null, + [OPENAI_RESPONSES_STREAMED_USAGE]: null, 'response.usage.billable': [], 'response.http.headers': [], }); @@ -261,7 +261,7 @@ const callResponsesCompactUpstream = defineStage< message: text, ...(parsed === undefined ? {} : { body: parsed }), }, - [RESPONSES_STREAMED_USAGE]: null, + [OPENAI_RESPONSES_STREAMED_USAGE]: null, // An upstream that was called and reported nothing, which is a different statement // from reporting zero. 'response.usage.billable': [{ identity, quantities: {} }], @@ -269,21 +269,21 @@ const callResponsesCompactUpstream = defineStage< }); } if (result.action !== 'compact') { - throw new Error(`callResponsesCompactUpstream: ${facts['route.attempt'].upstreamId} answered a compaction dial with a ${result.action} turn`); + throw new Error(`callOpenAIResponsesCompactUpstream: ${facts['route.attempt'].upstreamId} answered a compaction dial with a ${result.action} turn`); } // This candidate answered, so it is the one a follow-up turn carrying our own state must // come back to. use.selectAffinity(candidate); - const billable = [billedResponsesEntity(identity, billableUsageFromResponsesResult(result.result) ?? undefined)]; + const billable = [billedOpenAIResponsesEntity(identity, billableUsageFromOpenAIResponsesResult(result.result) ?? undefined)]; return move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream' as const, frames: syntheticEventsFromCompaction(result.result) }, // A compaction states its own counts, so what it billed is known before the frames are // read. It travels as a reading still to come rather than as one already settled, // because the verdict that goes with it is the edge's to add. - [RESPONSES_STREAMED_USAGE]: Promise.resolve({ billable, failed: false }), + [OPENAI_RESPONSES_STREAMED_USAGE]: Promise.resolve({ billable, failed: false }), 'response.usage.billable': billable, 'response.http.headers': [], }); @@ -295,12 +295,12 @@ const callResponsesCompactUpstream = defineStage< const dialSummarizationWire = dialChatWire({ source: 'request.chat.openaiResponses', needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.openaiResponses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + provides: ['response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], pick: endpoints => openaiResponsesTarget.pick(endpoints), wire: openaiResponsesWireFor, }); -const compactionWire: ChatWire = compose('openaiResponsesCompactNative', [...openaiResponsesWireRules, callResponsesCompactUpstream]); +const compactionWire: ChatWire = compose('openaiResponsesCompactNative', [...openaiResponsesWireRules, callOpenAIResponsesCompactUpstream]); // The ending below picks this wire only for a candidate whose compactions are the shim's to // simulate, so the ask is already answered: every turn that reaches it is one to summarize. const simulationWire: ChatWire = compose('openaiResponsesCompactSimulated', [summarizeForCompaction(() => true), dialSummarizationWire]); @@ -317,14 +317,14 @@ const simulationWire: ChatWire = compose('openaiResponsesCompactSimulated', [sum * It is last, which is what earns it the right to name a target at all, and it holds no state * across candidates: failover re-running the suffix re-runs this stage. */ -const dialResponsesCompaction = defineStage< +const dialOpenAIResponsesCompaction = defineStage< R<'route.attempt'> & Record, Record, Record, Record, ChatServices >({ - name: 'dialResponsesCompaction', + name: 'dialOpenAIResponsesCompaction', into: { request: { needs: ['route.attempt', 'request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], @@ -339,7 +339,7 @@ const dialResponsesCompaction = defineStage< consumes: [], provides: [ 'response.chat.openaiResponses', - RESPONSES_STREAMED_USAGE, + OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers', ], @@ -362,13 +362,13 @@ export type OpenAIResponsesCompactExit = R< | 'response.http.status' | 'response.http.headers' >; -export const openaiResponsesCompactPipeline = (payload: CanonicalResponsesPayload): Pipeline => { +export const openaiResponsesCompactPipeline = (payload: CanonicalOpenAIResponsesPayload): Pipeline => { // One cell per run, written by the stage directly above the one that reads it — the same // hand-across the generate chain makes, for the same reason: the resolver takes its // narrowing at assembly, before any fact exists. let prepared = payload; return compose('openaiResponsesCompact', [ - emitResponsesCompaction, + emitOpenAIResponsesCompaction, writeSettlement( handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), handedUp => (handedUp as { 'response.chat.openaiResponses.streamedUsage'?: unknown })['response.chat.openaiResponses.streamedUsage'] !== null, @@ -382,6 +382,6 @@ export const openaiResponsesCompactPipeline = (payload: CanonicalResponsesPayloa materializeAttempt('request.chat.openaiResponses'), beginStoredAttempt, expandShimCompactions, - dialResponsesCompaction, + dialOpenAIResponsesCompaction, ]); }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/http.ts b/packages/gateway/src/data-plane/chat/openai-responses/http.ts index ba1738c424..b9ddd9ee3b 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/http.ts @@ -8,15 +8,15 @@ // answered by the chain on both routes rather than caught as a throw on either. import { openaiResponsesCompactPipeline } from './compact.ts'; -import { createResponsesHttpStore } from './items/store.ts'; +import { createOpenAIResponsesHttpStore } from './items/store.ts'; import { openaiResponsesServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { openChatPrologue } from '../prologue.ts'; import { move } from '@floway-dev/pipeline'; -import type { CanonicalResponsesPayload, OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; -import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; +import { canonicalizeOpenAIResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; /** The read, as a value rather than a throw, because a pipelined entry decides what to do * about a body it could not read before it opens a run rather than after one unwound. @@ -26,10 +26,10 @@ import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/ * can make: `input` and `input[0]` are different sentences, and a body with no `model` is * refused in words OpenAI's own clients already parse. */ const readRequest = (bytes: Uint8Array): - | { type: 'ok'; payload: CanonicalResponsesPayload } + | { type: 'ok'; payload: CanonicalOpenAIResponsesPayload } | { type: 'invalid'; message: string; param?: string; code?: string } => { try { - return { type: 'ok', payload: canonicalizeResponsesPayload(JSON.parse(new TextDecoder().decode(bytes)) as OpenAIResponsesRequestPayload) }; + return { type: 'ok', payload: canonicalizeOpenAIResponsesPayload(JSON.parse(new TextDecoder().decode(bytes)) as OpenAIResponsesRequestPayload) }; } catch (error) { if (!(error instanceof TranslatorInputError)) { return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; @@ -78,7 +78,7 @@ export const openaiResponsesHttp = { const prologue = openChatPrologue(c, ingress, { wantsStream, model: payload.model, - storeFactory: (apiKey, requestStartedAt) => createResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined), + storeFactory: (apiKey, requestStartedAt) => createOpenAIResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined), }); return await serveThrough( @@ -87,7 +87,7 @@ export const openaiResponsesHttp = { openaiResponsesServePipeline(payload), move({ 'ingress.http.headers': prologue.headers, - 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.sourceProtocol': 'openaiResponses', 'ingress.chat.openaiResponses.wantsStream': wantsStream, 'request.chat.openaiResponses': payload, 'serve.model': payload.model, @@ -110,7 +110,7 @@ export const openaiResponsesHttp = { const prologue = openChatPrologue(c, ingress, { wantsStream: false, model: payload.model, - storeFactory: (apiKey, requestStartedAt) => createResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined), + storeFactory: (apiKey, requestStartedAt) => createOpenAIResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined), }); return await serveThrough( @@ -119,7 +119,7 @@ export const openaiResponsesHttp = { openaiResponsesCompactPipeline(payload), move({ 'ingress.http.headers': prologue.headers, - 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.sourceProtocol': 'openaiResponses', 'request.chat.openaiResponses': payload, 'serve.model': payload.model, }) as never, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize.ts deleted file mode 100644 index 922e47a92d..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize.ts +++ /dev/null @@ -1,44 +0,0 @@ -// DeepSeek wire-dialect normalizer for the OpenAI Responses protocol. Always- -// attached; flag-gated by `vendor-deepseek`. Runs last among interceptors -// so it has the final say on the outbound wire body. -// -// Outbound (request → upstream): -// -// - `reasoning.effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel (produced when an Anthropic Messages source had `thinking: { type: -// 'disabled' }`, etc.). DeepSeek uses a top-level -// `thinking: { type: 'disabled' }` field instead. We strip the entire -// `reasoning` object and emit the DeepSeek form. -// -// Inbound: nothing today — the OpenAI-Responses-target dialect quirks that exist -// on OpenAI Chat Completions (assistant `reasoning_content` field, `prompt_cache_*_tokens` -// usage) have no OpenAI-Responses-shape equivalent that has surfaced. Add hooks -// here if vendor-specific OpenAI Responses inbound rewrites become necessary. -// -// Reference: -// - https://api-docs.deepseek.com/zh-cn/guides/thinking_mode - -import type { OpenAIResponsesInterceptor } from './types.ts'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; - -interface DeepSeekDisableField { - thinking?: { type: 'disabled' }; -} - -type CanonicalOpenAIResponsesPayloadWithDeepSeekDisable = Omit & DeepSeekDisableField; - -const stripCanonicalReasoningSentinel = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { - if (payload.reasoning?.effort !== 'none') return payload; - const { reasoning: _stripped, ...rest } = payload; - const out: CanonicalOpenAIResponsesPayloadWithDeepSeekDisable = { ...rest, thinking: { type: 'disabled' } }; - return out as CanonicalOpenAIResponsesPayload; -}; - -export const withVendorDeepSeekOpenAIResponsesNormalize: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-deepseek')) return await run(); - - ctx.payload = stripCanonicalReasoningSentinel(ctx.payload); - - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts index e8e52776c3..34f3d1dc17 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts @@ -1,40 +1,40 @@ -import { createResponsesStorageKey, hashResponsesItem, openaiResponsesItemId } from './identity.ts'; +import { createOpenAIResponsesStorageKey, hashOpenAIResponsesItem, openaiResponsesItemId } from './identity.ts'; import { getRepo } from '../../../../repo/index.ts'; -import { assertSameStoredResponsesItem, cloneStoredResponsesItem, cloneStoredResponsesSnapshot, compareResponsesItemsByFreshness, scopedResponsesKey } from '../../../../repo/openai-responses-clone.ts'; -import { quantizeResponsesRefreshedAt, openaiResponsesStateCutoff } from '../../../../repo/openai-responses-retention.ts'; -import type { ApiKey, Repo, StoredResponsesItem, StoredResponsesSnapshot } from '../../../../repo/types.ts'; +import { assertSameStoredOpenAIResponsesItem, cloneStoredOpenAIResponsesItem, cloneStoredOpenAIResponsesSnapshot, compareOpenAIResponsesItemsByFreshness, scopedOpenAIResponsesKey } from '../../../../repo/openai-responses-clone.ts'; +import { quantizeOpenAIResponsesRefreshedAt, openaiResponsesStateCutoff } from '../../../../repo/openai-responses-retention.ts'; +import type { ApiKey, Repo, StoredOpenAIResponsesItem, StoredOpenAIResponsesSnapshot } from '../../../../repo/types.ts'; import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; -interface StatefulResponsesItemLookup { +interface StatefulOpenAIResponsesItemLookup { readonly apiKeyId: string; readonly ids: readonly string[]; readonly itemHashes: readonly string[]; } -interface StatefulResponsesBacking { - lookupItems(query: StatefulResponsesItemLookup): Promise; - insertItems(items: readonly StoredResponsesItem[]): Promise; - refreshItems(items: readonly StoredResponsesItem[], refreshedAt: number): Promise; - lookupSnapshot(apiKeyId: string, id: string): Promise; - insertSnapshot(snapshot: StoredResponsesSnapshot): Promise; +interface StatefulOpenAIResponsesBacking { + lookupItems(query: StatefulOpenAIResponsesItemLookup): Promise; + insertItems(items: readonly StoredOpenAIResponsesItem[]): Promise; + refreshItems(items: readonly StoredOpenAIResponsesItem[], refreshedAt: number): Promise; + lookupSnapshot(apiKeyId: string, id: string): Promise; + insertSnapshot(snapshot: StoredOpenAIResponsesSnapshot): Promise; } -interface LayeredStatefulResponsesStoreOptions { +interface LayeredStatefulOpenAIResponsesStoreOptions { readonly apiKeyId: string; - readonly reads: readonly StatefulResponsesBacking[]; - readonly writes: readonly StatefulResponsesBacking[]; + readonly reads: readonly StatefulOpenAIResponsesBacking[]; + readonly writes: readonly StatefulOpenAIResponsesBacking[]; } type OpenAIResponsesSnapshotMode = 'append' | 'replace'; -export interface StatefulResponsesStore { +export interface StatefulOpenAIResponsesStore { readonly apiKeyId: string; readonly writesState: boolean; - loadSnapshot(id: string): Promise; + loadSnapshot(id: string): Promise; loadInputItems(sourceItems: readonly OpenAIResponsesInputItem[], inputItemsToStage: readonly OpenAIResponsesInputItem[]): Promise; - getItemById(id: string): StoredResponsesItem | undefined; + getItemById(id: string): StoredOpenAIResponsesItem | undefined; stageInputItems(items: readonly OpenAIResponsesInputItem[]): Promise; - persistOutputItem(row: StoredResponsesItem): Promise; + persistOutputItem(row: StoredOpenAIResponsesItem): Promise; commitSnapshot(responseId: string, mode: OpenAIResponsesSnapshotMode, outputItemIds: readonly string[]): Promise; // Per-attempt transient state. `beginAttempt` reseeds the private-payload // scratchpad from hydrated items; interceptors can add server-only state @@ -44,16 +44,16 @@ export interface StatefulResponsesStore { getPrivatePayload(id: string): unknown; } -export class LayeredStatefulResponsesStore implements StatefulResponsesStore { - private readonly loadedItems = new Map(); - private readonly loadedByItemHash = new Map(); +export class LayeredStatefulOpenAIResponsesStore implements StatefulOpenAIResponsesStore { + private readonly loadedItems = new Map(); + private readonly loadedByItemHash = new Map(); private readonly stagedInputItemIds: string[] = []; private previousSnapshotItemIds: string[] = []; private readonly committedItemIds = new Set(); private readonly privatePayloads = new Map(); private readonly inputItemHashes = new WeakMap(); - constructor(private readonly options: LayeredStatefulResponsesStoreOptions) {} + constructor(private readonly options: LayeredStatefulOpenAIResponsesStoreOptions) {} get apiKeyId(): string { return this.options.apiKeyId; @@ -63,14 +63,14 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { return this.options.writes.length > 0; } - async loadSnapshot(id: string): Promise { + async loadSnapshot(id: string): Promise { for (const backing of this.options.reads) { const snapshot = await backing.lookupSnapshot(this.apiKeyId, id); if (snapshot === null) continue; await this.loadItems({ ids: snapshot.itemIds, itemHashes: [] }); if (!snapshot.itemIds.every(itemId => this.loadedItems.has(itemId))) continue; if (this.options.writes.length > 0) { - const refreshedAt = quantizeResponsesRefreshedAt(Date.now()); + const refreshedAt = quantizeOpenAIResponsesRefreshedAt(Date.now()); const items = snapshot.itemIds.map(itemId => this.loadedItems.get(itemId)!); await this.commitItems(items); const staleItems = items.filter(item => item.refreshedAt < refreshedAt); @@ -84,7 +84,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { if (snapshot.refreshedAt < refreshedAt) snapshot.refreshedAt = refreshedAt; } this.previousSnapshotItemIds = [...snapshot.itemIds]; - return cloneStoredResponsesSnapshot(snapshot); + return cloneStoredOpenAIResponsesSnapshot(snapshot); } return null; } @@ -107,9 +107,9 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { await this.loadItems({ ids: [...ids], itemHashes: [...itemHashes] }); } - getItemById(id: string): StoredResponsesItem | undefined { + getItemById(id: string): StoredOpenAIResponsesItem | undefined { const row = this.loadedItems.get(id); - return row === undefined ? undefined : cloneStoredResponsesItem(row); + return row === undefined ? undefined : cloneStoredOpenAIResponsesItem(row); } async stageInputItems(items: readonly OpenAIResponsesInputItem[]): Promise { @@ -117,11 +117,11 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { for (const item of items) await this.stageInputItem(item); } - async persistOutputItem(row: StoredResponsesItem): Promise { + async persistOutputItem(row: StoredOpenAIResponsesItem): Promise { if (!this.writesState) return; - const cloned = cloneStoredResponsesItem({ + const cloned = cloneStoredOpenAIResponsesItem({ ...row, - refreshedAt: quantizeResponsesRefreshedAt(row.refreshedAt), + refreshedAt: quantizeOpenAIResponsesRefreshedAt(row.refreshedAt), }); await this.commitItems([cloned]); this.rememberItem(cloned); @@ -139,7 +139,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { return row; }); await this.commitItems(uniqueRows); - const refreshedAt = quantizeResponsesRefreshedAt(Date.now()); + const refreshedAt = quantizeOpenAIResponsesRefreshedAt(Date.now()); const staleRows = uniqueRows.filter(row => row.refreshedAt < refreshedAt); if (staleRows.length > 0) { await Promise.all(this.options.writes.map(write => write.refreshItems(staleRows, refreshedAt))); @@ -148,7 +148,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { } } const snapshotRefreshedAt = Math.min(...uniqueRows.map(row => row.refreshedAt)); - const snapshot: StoredResponsesSnapshot = { + const snapshot: StoredOpenAIResponsesSnapshot = { id: responseId, apiKeyId: this.apiKeyId, itemIds, @@ -197,12 +197,12 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { return; } - const created: StoredResponsesItem = { + const created: StoredOpenAIResponsesItem = { id, apiKeyId: this.apiKeyId, payload: { item }, itemHash: await this.hashInputItem(item), - refreshedAt: quantizeResponsesRefreshedAt(Date.now()), + refreshedAt: quantizeOpenAIResponsesRefreshedAt(Date.now()), }; this.stagedInputItemIds.push(id); this.rememberItem(created); @@ -216,12 +216,12 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { return; } - const row: StoredResponsesItem = { - id: createResponsesStorageKey(), + const row: StoredOpenAIResponsesItem = { + id: createOpenAIResponsesStorageKey(), apiKeyId: this.apiKeyId, payload: { item }, itemHash, - refreshedAt: quantizeResponsesRefreshedAt(Date.now()), + refreshedAt: quantizeOpenAIResponsesRefreshedAt(Date.now()), }; this.stagedInputItemIds.push(row.id); this.rememberItem(row); @@ -230,28 +230,28 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { private async hashInputItem(item: OpenAIResponsesInputItem): Promise { const cached = this.inputItemHashes.get(item); if (cached !== undefined) return cached; - const hash = await hashResponsesItem(item); + const hash = await hashOpenAIResponsesItem(item); this.inputItemHashes.set(item, hash); return hash; } - private rememberItem(row: StoredResponsesItem): void { - const cloned = cloneStoredResponsesItem(row); + private rememberItem(row: StoredOpenAIResponsesItem): void { + const cloned = cloneStoredOpenAIResponsesItem(row); const existing = this.loadedItems.get(cloned.id); if (existing !== undefined && existing.refreshedAt >= cloned.refreshedAt) return; this.loadedItems.set(cloned.id, cloned); const byHash = this.loadedByItemHash.get(cloned.itemHash); - if (byHash === undefined || compareResponsesItemsByFreshness(cloned, byHash) < 0) { + if (byHash === undefined || compareOpenAIResponsesItemsByFreshness(cloned, byHash) < 0) { this.loadedByItemHash.set(cloned.itemHash, cloned); } } - private async commitItems(rows: readonly StoredResponsesItem[]): Promise { + private async commitItems(rows: readonly StoredOpenAIResponsesItem[]): Promise { const pending = rows.flatMap(row => { if (!this.committedItemIds.has(row.id)) return [row]; const committed = this.loadedItems.get(row.id); if (committed === undefined) throw new Error(`Committed Responses item disappeared from request state: ${row.id}`); - assertSameStoredResponsesItem(row, committed); + assertSameStoredOpenAIResponsesItem(row, committed); return []; }); if (pending.length === 0) return; @@ -260,7 +260,7 @@ export class LayeredStatefulResponsesStore implements StatefulResponsesStore { } } -export class RepoStatefulResponsesBacking implements StatefulResponsesBacking { +export class RepoStatefulOpenAIResponsesBacking implements StatefulOpenAIResponsesBacking { private readonly earliestVisibleCutoff: number; constructor( @@ -271,116 +271,116 @@ export class RepoStatefulResponsesBacking implements StatefulResponsesBacking { this.earliestVisibleCutoff = openaiResponsesStateCutoff(requestStartedAt, retentionSeconds); } - async lookupItems(query: StatefulResponsesItemLookup): Promise { + async lookupItems(query: StatefulOpenAIResponsesItemLookup): Promise { const [byId, byItemHash] = await Promise.all([ this.getRepo().openaiResponsesItems.lookupMany(query.apiKeyId, query.ids, this.earliestVisibleCutoff), this.getRepo().openaiResponsesItems.lookupManyByItemHash(query.apiKeyId, query.itemHashes, this.earliestVisibleCutoff), ]); - const rows = new Map(); - for (const row of [...byId, ...byItemHash]) rows.set(scopedResponsesKey(row.apiKeyId, row.id), row); + const rows = new Map(); + for (const row of [...byId, ...byItemHash]) rows.set(scopedOpenAIResponsesKey(row.apiKeyId, row.id), row); return [...rows.values()]; } - async insertItems(items: readonly StoredResponsesItem[]): Promise { + async insertItems(items: readonly StoredOpenAIResponsesItem[]): Promise { await this.getRepo().openaiResponsesItems.insertMany(items, this.earliestVisibleCutoff); } - async refreshItems(items: readonly StoredResponsesItem[], refreshedAt: number): Promise { + async refreshItems(items: readonly StoredOpenAIResponsesItem[], refreshedAt: number): Promise { await this.getRepo().openaiResponsesItems.refreshMany(items, refreshedAt, this.earliestVisibleCutoff); } - async lookupSnapshot(apiKeyId: string, id: string): Promise { + async lookupSnapshot(apiKeyId: string, id: string): Promise { return await this.getRepo().openaiResponsesSnapshots.lookup(apiKeyId, id, this.earliestVisibleCutoff); } - async insertSnapshot(snapshot: StoredResponsesSnapshot): Promise { + async insertSnapshot(snapshot: StoredOpenAIResponsesSnapshot): Promise { await this.getRepo().openaiResponsesSnapshots.insert(snapshot); } } -export class MemoryStatefulResponsesBacking implements StatefulResponsesBacking { - private readonly items = new Map(); - private readonly snapshots = new Map(); +export class MemoryStatefulOpenAIResponsesBacking implements StatefulOpenAIResponsesBacking { + private readonly items = new Map(); + private readonly snapshots = new Map(); - lookupItems(query: StatefulResponsesItemLookup): Promise { + lookupItems(query: StatefulOpenAIResponsesItemLookup): Promise { const ids = new Set(query.ids); const hashes = new Set(query.itemHashes); return Promise.resolve([...this.items.values()] .filter(row => row.apiKeyId === query.apiKeyId && (ids.has(row.id) || hashes.has(row.itemHash))) - .map(cloneStoredResponsesItem) - .toSorted(compareResponsesItemsByFreshness)); + .map(cloneStoredOpenAIResponsesItem) + .toSorted(compareOpenAIResponsesItemsByFreshness)); } - async insertItems(items: readonly StoredResponsesItem[]): Promise { + async insertItems(items: readonly StoredOpenAIResponsesItem[]): Promise { const quantizedItems = items.map(item => ({ ...item, - refreshedAt: quantizeResponsesRefreshedAt(item.refreshedAt), + refreshedAt: quantizeOpenAIResponsesRefreshedAt(item.refreshedAt), })); - const pending = new Map(); + const pending = new Map(); for (const item of quantizedItems) { - const key = scopedResponsesKey(item.apiKeyId, item.id); + const key = scopedOpenAIResponsesKey(item.apiKeyId, item.id); const existing = pending.get(key) ?? this.items.get(key); - if (existing !== undefined) assertSameStoredResponsesItem(item, existing); + if (existing !== undefined) assertSameStoredOpenAIResponsesItem(item, existing); else pending.set(key, item); } - for (const [key, item] of pending) this.items.set(key, cloneStoredResponsesItem(item)); + for (const [key, item] of pending) this.items.set(key, cloneStoredOpenAIResponsesItem(item)); for (const item of quantizedItems) { - const stored = this.items.get(scopedResponsesKey(item.apiKeyId, item.id))!; + const stored = this.items.get(scopedOpenAIResponsesKey(item.apiKeyId, item.id))!; if (stored.refreshedAt < item.refreshedAt) stored.refreshedAt = item.refreshedAt; } } - async refreshItems(items: readonly StoredResponsesItem[], refreshedAt: number): Promise { - const quantizedRefreshedAt = quantizeResponsesRefreshedAt(refreshedAt); - const existing = items.map(item => this.items.get(scopedResponsesKey(item.apiKeyId, item.id))); + async refreshItems(items: readonly StoredOpenAIResponsesItem[], refreshedAt: number): Promise { + const quantizedRefreshedAt = quantizeOpenAIResponsesRefreshedAt(refreshedAt); + const existing = items.map(item => this.items.get(scopedOpenAIResponsesKey(item.apiKeyId, item.id))); const missingIndex = existing.findIndex(item => item === undefined); if (missingIndex !== -1) { throw new Error(`Responses item disappeared before retention refresh: ${items[missingIndex].id}`); } for (let index = 0; index < existing.length; index += 1) { - assertSameStoredResponsesItem(items[index], existing[index]!); + assertSameStoredOpenAIResponsesItem(items[index], existing[index]!); if (existing[index]!.refreshedAt < quantizedRefreshedAt) existing[index]!.refreshedAt = quantizedRefreshedAt; } } - lookupSnapshot(apiKeyId: string, id: string): Promise { - const snapshot = this.snapshots.get(scopedResponsesKey(apiKeyId, id)); - return Promise.resolve(snapshot === undefined ? null : cloneStoredResponsesSnapshot(snapshot)); + lookupSnapshot(apiKeyId: string, id: string): Promise { + const snapshot = this.snapshots.get(scopedOpenAIResponsesKey(apiKeyId, id)); + return Promise.resolve(snapshot === undefined ? null : cloneStoredOpenAIResponsesSnapshot(snapshot)); } - insertSnapshot(snapshot: StoredResponsesSnapshot): Promise { + insertSnapshot(snapshot: StoredOpenAIResponsesSnapshot): Promise { const quantized = { ...snapshot, - refreshedAt: quantizeResponsesRefreshedAt(snapshot.refreshedAt), + refreshedAt: quantizeOpenAIResponsesRefreshedAt(snapshot.refreshedAt), }; - const key = scopedResponsesKey(quantized.apiKeyId, quantized.id); + const key = scopedOpenAIResponsesKey(quantized.apiKeyId, quantized.id); const existing = this.snapshots.get(key); if (existing === undefined || quantized.refreshedAt > existing.refreshedAt) { - this.snapshots.set(key, cloneStoredResponsesSnapshot(quantized)); + this.snapshots.set(key, cloneStoredOpenAIResponsesSnapshot(quantized)); } return Promise.resolve(); } - // Beyond `StatefulResponsesBacking`: the spec scopes eviction to the + // Beyond `StatefulOpenAIResponsesBacking`: the spec scopes eviction to the // connection-local cache, so the delete path deliberately stops at this // in-memory backing rather than becoming a contract every backing — the // durable one included — has to answer for. // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L127 evictSnapshot(apiKeyId: string, id: string): void { - this.snapshots.delete(scopedResponsesKey(apiKeyId, id)); + this.snapshots.delete(scopedOpenAIResponsesKey(apiKeyId, id)); } } type OpenAIResponsesStatePolicy = Pick; -const createDurableBacking = (apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number): RepoStatefulResponsesBacking | null => +const createDurableBacking = (apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number): RepoStatefulOpenAIResponsesBacking | null => apiKey.openaiResponsesRetentionSeconds === 0 ? null - : new RepoStatefulResponsesBacking(getRepo, requestStartedAt, apiKey.openaiResponsesRetentionSeconds); + : new RepoStatefulOpenAIResponsesBacking(getRepo, requestStartedAt, apiKey.openaiResponsesRetentionSeconds); -export const createResponsesHttpStore = (apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore => { +export const createOpenAIResponsesHttpStore = (apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulOpenAIResponsesStore => { const durable = createDurableBacking(apiKey, requestStartedAt); - return new LayeredStatefulResponsesStore({ + return new LayeredStatefulOpenAIResponsesStore({ apiKeyId: apiKey.id, reads: durable === null ? [] : [durable], writes: durable === null || store === false ? [] : [durable], @@ -393,11 +393,11 @@ export const createResponsesHttpStore = (apiKey: OpenAIResponsesStatePolicy, req // it is what will write there once that shim is a stage. So they get a store // with no backing: it holds per-attempt state in memory and reads/writes nothing // durable, keeping the store present on every chat ctx. -export const createNonResponsesSourceStore = (apiKeyId: string): StatefulResponsesStore => - new LayeredStatefulResponsesStore({ apiKeyId, reads: [], writes: [] }); +export const createNonOpenAIResponsesSourceStore = (apiKeyId: string): StatefulOpenAIResponsesStore => + new LayeredStatefulOpenAIResponsesStore({ apiKeyId, reads: [], writes: [] }); -export const createResponsesWsSession = (): { - createStore(apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore; +export const createOpenAIResponsesWsSession = (): { + createStore(apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulOpenAIResponsesStore; evictSnapshot(apiKeyId: string, id: string): void; } => { // "Servers SHOULD keep the most recent previous-response state in @@ -406,14 +406,14 @@ export const createResponsesWsSession = (): { // available from connection-local state, the server MUST fail the turn with // an error whose code is `previous_response_not_found`." // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L125 - const local = new MemoryStatefulResponsesBacking(); + const local = new MemoryStatefulOpenAIResponsesBacking(); return { - createStore(apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulResponsesStore { + createStore(apiKey: OpenAIResponsesStatePolicy, requestStartedAt: number, store: boolean | undefined): StatefulOpenAIResponsesStore { const durable = createDurableBacking(apiKey, requestStartedAt); // Session-local state is the first store:true collision gate. Writing it // first prevents a rejected local history from creating a durable row. const writes = store === false || durable === null ? [local] : [local, durable]; - return new LayeredStatefulResponsesStore({ + return new LayeredStatefulOpenAIResponsesStore({ apiKeyId: apiKey.id, reads: durable === null ? [local] : [local, durable], writes, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index a19f500bf9..191d61ad77 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -1,6 +1,6 @@ // Responses as a pipeline, on the chain Chat Completions established. // -// emitResponses the edge: writes the answer in the shape the client asked for +// emitOpenAIResponses the edge: writes the answer in the shape the client asked for // writeSettlement above the fork, so a run bills once however many wires it tried // hydrateStoredItems the stored-items membrane, on the way in // resolveChatCandidates narrows to what can serve, in the order affinity asks for @@ -15,7 +15,7 @@ // translated ones — Responses via Messages and Responses via Chat Completions — each a // handoff followed by that protocol's own wire. What sits *in* a wire rather than above the // fork is a rule that speaks about that wire: the role rewrite and the cache-bucket fold both -// do, which is why their interceptor forms stood down on `ctx.targetApi !== 'responses'`, and +// do, which is why their interceptor forms stood down on `ctx.targetApi !== 'openaiResponses'`, and // position says it here instead. // // Two transports enter here — `POST /v1/responses` and the WebSocket one — and the only thing @@ -33,7 +33,7 @@ // `generate`; what the ending answers with is the branch the provider says it ran, which // is why the envelope a compaction is arrives somewhere rather than nowhere. // - the server-tool shims. They never became stages and the interceptor array that ran them -// is gone, so `withResponsesServerToolShim` and the two tools under it are reachable only +// is gone, so `withOpenAIResponsesServerToolShim` and the two tools under it are reachable only // from their own tests: a turn declaring a hosted `web_search` or `image_generation` tool // reaches the upstream unshimmed, and the two flags that gate them do nothing. The code is // kept so that porting it starts from something rather than from nothing. Nothing in a @@ -44,8 +44,8 @@ // pipeline: an upstream that refused is answered in its own words, with the status it sent, // rather than being quoted back inside an envelope this gateway wrote. -import { analyzeResponsesAffinity } from './affinity/ingress.ts'; -import { wrapResponsesClientEgress } from './client-output.ts'; +import { analyzeOpenAIResponsesAffinity } from './affinity/ingress.ts'; +import { wrapOpenAIResponsesClientEgress } from './client-output.ts'; import { buildCompactionEnvelope, containsCompactionTrigger, @@ -55,11 +55,11 @@ import { summaryTextFrom, } from './compact-shim.ts'; import type { OpenAIResponsesServeFailure } from './errors.ts'; -import { hydrateResponsesPayload } from './items/hydrate.ts'; +import { hydrateOpenAIResponsesPayload } from './items/hydrate.ts'; import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; import { syntheticEventsFromResult } from './items/output.ts'; import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve-prep.ts'; -import { billableUsageFromResponsesEvent, billableUsageFromResponsesResult } from './usage.ts'; +import { billableUsageFromOpenAIResponsesEvent, billableUsageFromOpenAIResponsesResult } from './usage.ts'; import { recordFrames } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { AttemptSelector, BillableEntity } from '../../pipeline/facts.ts'; @@ -75,15 +75,15 @@ import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.t import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { - applyRoleCompatibilityToResponses, - disableReasoningOnForcedToolChoiceForResponses, - normalizeExclusiveCachedTokensForResponses, - stripPromptCacheKeyForResponses, - vendorDeepSeekNormalizeForResponses, - vendorQwenNormalizeForResponses, + applyRoleCompatibilityToOpenAIResponses, + disableReasoningOnForcedToolChoiceForOpenAIResponses, + normalizeExclusiveCachedTokensForOpenAIResponses, + stripPromptCacheKeyForOpenAIResponses, + vendorDeepSeekNormalizeForOpenAIResponses, + vendorQwenNormalizeForOpenAIResponses, } from '../interceptors.ts'; import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; -import { applyRulesToUpstreamResponses } from '../shared/alias-rules.ts'; +import { applyRulesToUpstreamOpenAIResponses } from '../shared/alias-rules.ts'; import { tryCatchChatServeFailure } from '../shared/errors.ts'; import { createExternalImageLoader } from '../shared/external-image-loader.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; @@ -92,23 +92,23 @@ import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type Cha import { compose, defineStage, move, type Pipeline, type Stage, type Use } from '@floway-dev/pipeline'; import { doneFrame, eventFrame, sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { - collectResponsesProtocolEventsToResult, - createRandomResponsesItemId, - isResponsesTerminalEvent, + collectOpenAIResponsesProtocolEventsToResult, + createRandomOpenAIResponsesItemId, + isOpenAIResponsesTerminalEvent, openaiResponsesProtocolFrameToSSEFrame, - RESPONSES_MISSING_TERMINAL_MESSAGE, - type CanonicalResponsesPayload, + OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, + type CanonicalOpenAIResponsesPayload, type ClientResponseResource, - type ClientResponsesStreamEvent, + type ClientOpenAIResponsesStreamEvent, type OpenAIResponsesOutputItem, type OpenAIResponsesStreamEvent, } from '@floway-dev/protocols/openai-responses'; import { providerModelOf, toInternalDebugError, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; -import { translateResponsesViaChatCompletions, translateResponsesViaMessages } from '@floway-dev/translate'; +import { translateOpenAIResponsesViaOpenAIChatCompletions, translateOpenAIResponsesViaAnthropicMessages } from '@floway-dev/translate'; /** `/v1/responses` prefers its own wire, then the translated Messages path, then the * translated Chat Completions path. */ -export const openaiResponsesTarget = chatTargetPicker(['responses', 'messages', 'chat-completions']); +export const openaiResponsesTarget = chatTargetPicker(['openaiResponses', 'anthropicMessages', 'openaiChatCompletions']); /** How the transport that opened the run frames a streamed answer. Both carry this * protocol's own events and neither adds or drops one: SSE is the format an HTTP body is @@ -129,7 +129,7 @@ export interface OpenAIResponsesFacts extends ChatFacts { /** What the client is actually sent — an object when it asked for one, and the stream it * asked to be streamed, in the framing its transport writes. The edge provides it, so a * dump shows what the client received. */ - 'response.chat.openaiResponses.rendered': Record | AsyncIterable | AsyncIterable>; + 'response.chat.openaiResponses.rendered': Record | AsyncIterable | AsyncIterable>; /** What the upstream will have reported once the frames run out, and `null` when nothing * streamed. Settling from this is the epilogue's job, after the drain. */ 'response.chat.openaiResponses.streamedUsage': Promise | null; @@ -155,14 +155,14 @@ type R = { [P in K]: OpenAIResponsesFacts[ * Only the last step differs: `renderSSE` is a wire format written over an HTTP body, and a * transport that frames each event itself takes the events it would have been written from. */ -const emitResponses = (client: CanonicalResponsesPayload, framing: OpenAIResponsesStreamFraming) => defineStage< +const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, framing: OpenAIResponsesStreamFraming) => defineStage< R<'ingress.chat.openaiResponses.wantsStream'>, R<'ingress.chat.openaiResponses.wantsStream'>, R<'ingress.chat.openaiResponses.wantsStream' | 'response.chat.openaiResponses' | 'response.http.headers'>, R<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ - name: 'emitResponses', + name: 'emitOpenAIResponses', through: { request: { needs: ['ingress.chat.openaiResponses.wantsStream'], consumes: [], provides: [] }, response: { @@ -209,7 +209,7 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: OpenAIRespons // id beneath one response id this gateway minted, and the resource completed to what the // schema requires of it. It runs before the fold rather than beside it, so a client that // did not ask to stream is answered with the object the persisted frames add up to. - const egress = wrapResponsesClientEgress( + const egress = wrapOpenAIResponsesClientEgress( answer.frames as AsyncIterable>, use.gateway, client, @@ -230,7 +230,7 @@ const emitResponses = (client: CanonicalResponsesPayload, framing: OpenAIRespons ...rest, 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move( - await collectResponsesProtocolEventsToResult(frames) as unknown as Record, + await collectOpenAIResponsesProtocolEventsToResult(frames) as unknown as Record, ), 'response.http.status': 200, }; @@ -278,7 +278,7 @@ export const internalErrorEnvelope = (error: unknown): Record = * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L170-L177 * https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/core/streaming.ts#L69-L71 * https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/_streaming.py#L87-L98 */ -const streamErrorEvent = (error: unknown): ClientResponsesStreamEvent => { +const streamErrorEvent = (error: unknown): ClientOpenAIResponsesStreamEvent => { const debug = toInternalDebugError(error); return { type: 'error', @@ -290,17 +290,17 @@ const streamErrorEvent = (error: unknown): ClientResponsesStreamEvent => { cause: debug.cause, target_api: debug.target_api, }, - } as unknown as ClientResponsesStreamEvent; + } as unknown as ClientOpenAIResponsesStreamEvent; }; /** "Any error incurred while streaming will be followed by a `response.failed` event." * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L430 */ -const streamFailedEvent = (announced: ClientResponseResource, error: unknown): ClientResponsesStreamEvent => { +const streamFailedEvent = (announced: ClientResponseResource, error: unknown): ClientOpenAIResponsesStreamEvent => { const debug = toInternalDebugError(error); return { type: 'response.failed', response: { ...announced, status: 'failed', error: { code: debug.type, message: debug.message } }, - } as ClientResponsesStreamEvent; + } as ClientOpenAIResponsesStreamEvent; }; /** Every Responses event has an SSE form of its own, so the render is a straight map. What @@ -314,7 +314,7 @@ const streamFailedEvent = (announced: ClientResponseResource, error: unknown): C * a stream that ran out before saying how it ended. The status went out with the headers, so * the failure has to be said in the protocol's own words, and it is said *instead of* the * terminator: a stream that ended on `[DONE]` is a stream that finished. */ -const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ [Symbol.asyncIterator]: () => (async function* () { // The last resource the client was shown, which is the one a `response.failed` restates: // the id it names has to be the id the client already saw this turn under. @@ -344,12 +344,12 @@ const renderSSE = (frames: AsyncIterable defineStage< +const callOpenAIResponsesUpstream = (streamedUsage: string) => defineStage< R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, R<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'> & Record, ChatServices >({ - name: 'callResponsesUpstream', + name: 'callOpenAIResponsesUpstream', return: { provides: [ 'response.chat.openaiResponses', @@ -372,12 +372,12 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< // // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to, which is the one a wire takes. - const asked = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; - const body = bodyForAttempt(asked, candidate, applyRulesToUpstreamResponses); + const asked = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const body = bodyForAttempt(asked, candidate, applyRulesToUpstreamOpenAIResponses); let result; try { - result = await candidate.provider.instance.callResponses( + result = await candidate.provider.instance.callOpenAIResponses( providerModelOf(candidate), body, 'generate', @@ -436,12 +436,12 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< ...facts, 'response.chat.openaiResponses': { kind: 'value' as const, body: result.result }, [streamedUsage]: null, - 'response.usage.billable': [billedResponsesEntity(identity, billableUsageFromResponsesResult(result.result) ?? undefined)], + 'response.usage.billable': [billedOpenAIResponsesEntity(identity, billableUsageFromOpenAIResponsesResult(result.result) ?? undefined)], 'response.http.headers': [], }); } - const metered = meterResponses(result.events, identity, use.gateway.attempt); + const metered = meterOpenAIResponses(result.events, identity, use.gateway.attempt); return move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream' as const, frames: metered.frames }, @@ -464,20 +464,20 @@ const callResponsesUpstream = (streamedUsage: string) => defineStage< * on assistant content, so a turn that leaves for Messages or Chat Completions never carried * the disagreement in the first place. */ -const normalizeAssistantContentForResponses = defineStage< +const normalizeAssistantContentForOpenAIResponses = defineStage< R<'request.chat.openaiResponses'>, R<'request.chat.openaiResponses'>, Record, Record, ChatServices >({ - name: 'normalizeAssistantContentForResponses', + name: 'normalizeAssistantContentForOpenAIResponses', through: { request: { needs: ['request.chat.openaiResponses'], consumes: [], provides: ['request.chat.openaiResponses'] }, response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next) => { - const payload = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; const input = normalizeAssistantInputText(payload.input); // A rewrite that changed nothing hands the same payload on, so the record shows no // change where none happened. @@ -500,51 +500,51 @@ const normalizeAssistantContentForResponses = defineStage< * subject to the same three. */ export const openaiResponsesWireRules: readonly Stage[] = [ - normalizeAssistantContentForResponses, - disableReasoningOnForcedToolChoiceForResponses, - applyRoleCompatibilityToResponses, - stripPromptCacheKeyForResponses, - normalizeExclusiveCachedTokensForResponses, - vendorDeepSeekNormalizeForResponses, - vendorQwenNormalizeForResponses, + normalizeAssistantContentForOpenAIResponses, + disableReasoningOnForcedToolChoiceForOpenAIResponses, + applyRoleCompatibilityToOpenAIResponses, + stripPromptCacheKeyForOpenAIResponses, + normalizeExclusiveCachedTokensForOpenAIResponses, + vendorDeepSeekNormalizeForOpenAIResponses, + vendorQwenNormalizeForOpenAIResponses, ]; /** The Responses wire, as the chain that dials it. */ export const openaiResponsesWire = (streamedUsage: string): readonly Stage[] => [ ...openaiResponsesWireRules, - callResponsesUpstream(streamedUsage), + callOpenAIResponsesUpstream(streamedUsage), ]; /** This family's own reading, which every wire under it hands up. */ -export const RESPONSES_STREAMED_USAGE = 'response.chat.openaiResponses.streamedUsage'; +export const OPENAI_RESPONSES_STREAMED_USAGE = 'response.chat.openaiResponses.streamedUsage'; /** The three wires `/v1/responses` can be served on. Its own is the bare wire; each translated * one is a handoff and then the target protocol's own wire. */ export const openaiResponsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { switch (target) { - case 'responses': - return compose('openaiResponsesNative', openaiResponsesWire(RESPONSES_STREAMED_USAGE)); - case 'messages': + case 'openaiResponses': + return compose('openaiResponsesNative', openaiResponsesWire(OPENAI_RESPONSES_STREAMED_USAGE)); + case 'anthropicMessages': return compose('openaiResponsesViaAnthropicMessages', [ handOff({ from: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, - trip: async payload => await translateResponsesViaMessages(payload, { + trip: async payload => await translateOpenAIResponsesViaAnthropicMessages(payload, { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), }), }), - ...anthropicMessagesWire(RESPONSES_STREAMED_USAGE), + ...anthropicMessagesWire(OPENAI_RESPONSES_STREAMED_USAGE), ]); - case 'chat-completions': + case 'openaiChatCompletions': return compose('openaiResponsesViaOpenAIChatCompletions', [ handOff({ from: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, - trip: async payload => await translateResponsesViaChatCompletions(payload, { model: candidate.model.id }), + trip: async payload => await translateOpenAIResponsesViaOpenAIChatCompletions(payload, { model: candidate.model.id }), }), - ...openaiChatCompletionsWire(RESPONSES_STREAMED_USAGE), + ...openaiChatCompletionsWire(OPENAI_RESPONSES_STREAMED_USAGE), ]); } }; @@ -553,7 +553,7 @@ export const openaiResponsesWireFor = (target: ChatTargetApi, candidate: ModelCa * pass and the client's stream is what drives it. Responses states its counts on the * lifecycle envelopes, and only one carrying real counts replaces the running figure, so an * envelope that states none cannot wipe a good reading. */ -const meterResponses = ( +const meterOpenAIResponses = ( source: AsyncIterable>, identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, @@ -569,21 +569,21 @@ const meterResponses = ( for await (const frame of source) { // Time to first token is measured where the token is, which is the only place that // knows a frame carries generated content rather than the envelope around it. - if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'responses')) { + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'openaiResponses')) { attempt.firstOutputTokenAt = performance.now(); } if (frame.type !== 'event') { yield frame; continue; } - const usage = billableUsageFromResponsesEvent(frame.event); + const usage = billableUsageFromOpenAIResponsesEvent(frame.event); if (usage !== null) reported = usage; // Read off the frame rather than off having been resumed past it. The stage that // stores the turn's items stops reading at the terminal event, so the resumption // never comes: what this loop would learn from it is already true when the frame // arrives, and the turn ended the way the upstream said it did whether or not // anything downstream asked for another. - if (isResponsesTerminalEvent(frame.event)) sawTerminal = true; + if (isOpenAIResponsesTerminalEvent(frame.event)) sawTerminal = true; yield frame; // The turn is over, so there is nothing further to read. An upstream that holds the // connection open past its terminal event would otherwise hold the client's stream @@ -592,12 +592,12 @@ const meterResponses = ( } // Frames ran out with no terminal event, which is a turn nobody can answer from: the // response was never stated complete, incomplete or failed. - throw new Error(RESPONSES_MISSING_TERMINAL_MESSAGE); + throw new Error(OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE); } finally { // Reached however the frames ended — the terminal event, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle({ billable: [billedResponsesEntity(identity, reported)], failed: !sawTerminal }); + settle({ billable: [billedOpenAIResponsesEntity(identity, reported)], failed: !sawTerminal }); } })(); return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; @@ -611,7 +611,7 @@ const meterResponses = ( * because on this protocol it is not a quantity but a rate selector — `service_tier` states * the tier the turn was actually served at, and that is the pricing entry it is billed * under. */ -export const billedResponsesEntity = (identity: TelemetryModelIdentity, usage: BillableUsage | undefined): BillableEntity => { +export const billedOpenAIResponsesEntity = (identity: TelemetryModelIdentity, usage: BillableUsage | undefined): BillableEntity => { if (usage === undefined) return { identity, quantities: {} }; const measurement = tokenUsageMeasurement(tokenUsageFromBillableUsage(usage)); return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; @@ -626,9 +626,9 @@ export const billedResponsesEntity = (identity: TelemetryModelIdentity, usage: B * client never sent, and a turn is pinned by what its items carry. The narrowing is built * at assembly, before any fact exists, so the membrane hands the prepared payload across * through the run's own cell rather than through the record. */ -export const openaiResponsesNarrowing = (prepared: () => CanonicalResponsesPayload): ChatNarrowing> => ({ +export const openaiResponsesNarrowing = (prepared: () => CanonicalOpenAIResponsesPayload): ChatNarrowing> => ({ canServe: candidate => openaiResponsesTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeResponsesAffinity(prepared(), gateway.affinity.codec), + affinity: async gateway => await analyzeOpenAIResponsesAffinity(prepared(), gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /responses endpoint.`, refuse: (status, message, reason) => ({ 'response.chat.openaiResponses.streamedUsage': null, @@ -684,8 +684,8 @@ const refuseStoredItems = ( * it carries the `return` trait alongside `through`. */ export const hydrateStoredItems = ( - client: CanonicalResponsesPayload, - prepared: (payload: CanonicalResponsesPayload) => void, + client: CanonicalOpenAIResponsesPayload, + prepared: (payload: CanonicalOpenAIResponsesPayload) => void, ) => defineStage< R<'request.chat.openaiResponses'>, R<'request.chat.openaiResponses' | 'request.chat.openaiResponses.privatePayloads'>, @@ -715,9 +715,9 @@ export const hydrateStoredItems = ( const store = use.gateway.store; // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to. - const asked = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; + const asked = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; - let expanded: CanonicalResponsesPayload; + let expanded: CanonicalOpenAIResponsesPayload; try { expanded = await expandPreviousResponseId(asked, store); } catch (error) { @@ -735,9 +735,9 @@ export const hydrateStoredItems = ( // read by item hash as well, so a body repeated verbatim finds the row it already made. await store.loadInputItems(expanded.input, client.input); - let hydrated: ReturnType; + let hydrated: ReturnType; try { - hydrated = hydrateResponsesPayload(expanded, store); + hydrated = hydrateOpenAIResponsesPayload(expanded, store); } catch (error) { const failure = tryCatchChatServeFailure(error); if (failure?.kind !== 'item-not-found') throw error; @@ -811,7 +811,7 @@ export const beginStoredAttempt = defineStage< * would have produced for it. */ export const simulatesCompaction = (candidate: ModelCandidate, attempt: AttemptSelector): boolean => - openaiResponsesTarget.pick(candidate.model.endpoints) !== 'responses' + openaiResponsesTarget.pick(candidate.model.endpoints) !== 'openaiResponses' || attempt.flags.includes('responses-compact-shim'); /** @@ -848,7 +848,7 @@ export const expandShimCompactions = defineStage< } // The key holds what a client may send, whose `input` is a string or a list; this chain // runs on the canonical form the entry normalized it to. - const payload = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; const expanded = expandShimCompactionItems(payload); // A turn carrying none of ours hands the same payload on, so the record shows no change // where none happened. @@ -899,7 +899,7 @@ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< }, execute: async (facts, next, use) => { if (!asked(facts, use)) return await next(facts); - const payload = facts['request.chat.openaiResponses'] as CanonicalResponsesPayload; + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; const back = await next({ ...facts, 'request.chat.openaiResponses': move(summarizationTurnFor(payload)) }); const answer = back['response.chat.openaiResponses']; @@ -921,7 +921,7 @@ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< yield frame; } })(); - const collected = await collectResponsesProtocolEventsToResult(observed); + const collected = await collectOpenAIResponsesProtocolEventsToResult(observed); const summaryText = summaryTextFrom(closed, collected.output); if (summaryText.length === 0) { @@ -931,7 +931,7 @@ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< return { ...back, 'response.chat.openaiResponses': move({ status: 502, message: EMPTY_SUMMARY_MESSAGE }) }; } - const synthesized = buildCompactionEnvelope(createRandomResponsesItemId('compaction'), summaryText, collected); + const synthesized = buildCompactionEnvelope(createRandomOpenAIResponsesItemId('compaction'), summaryText, collected); return { ...back, 'response.chat.openaiResponses': move({ kind: 'stream' as const, frames: syntheticEventsFromResult(synthesized) }) }; }, }); @@ -946,7 +946,7 @@ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< * native Responses upstream does with it. */ const asksForCompaction: CompactionAsk = (facts, use) => - containsCompactionTrigger((facts['request.chat.openaiResponses'] as CanonicalResponsesPayload).input) + containsCompactionTrigger((facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload).input) && simulatesCompaction(use.resolveAttempt(facts['route.attempt']), facts['route.attempt']); export type OpenAIResponsesServeEntry = R< @@ -960,7 +960,7 @@ export type OpenAIResponsesServeExit = R< >; export const openaiResponsesServePipeline = ( - payload: CanonicalResponsesPayload, + payload: CanonicalOpenAIResponsesPayload, // SSE is what a run is written in when nothing else claims its frames, which is every // entry over an HTTP body; the WebSocket transport says so because it writes its own. framing: OpenAIResponsesStreamFraming = 'sse', @@ -971,7 +971,7 @@ export const openaiResponsesServePipeline = ( // is the whole of what is known about the turn. let prepared = payload; return compose('openaiResponsesServe', [ - emitResponses(payload, framing), + emitOpenAIResponses(payload, framing), writeSettlement( handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), handedUp => (handedUp as { 'response.chat.openaiResponses.streamedUsage'?: unknown })['response.chat.openaiResponses.streamedUsage'] !== null, @@ -989,7 +989,7 @@ export const openaiResponsesServePipeline = ( dialChatWire({ source: 'request.chat.openaiResponses', needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.openaiResponses', RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + provides: ['response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], pick: endpoints => openaiResponsesTarget.pick(endpoints), wire: openaiResponsesWireFor, }), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts b/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts index 7b7978f76d..37c51eda9b 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts @@ -1,5 +1,5 @@ -import type { StatefulResponsesStore } from './items/store.ts'; -import type { CanonicalResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { StatefulOpenAIResponsesStore } from './items/store.ts'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; // Thrown when a request names a `previous_response_id` that the store cannot // resolve. The stage that hydrates catches this and answers with the @@ -28,9 +28,9 @@ export class PreviousResponseNotFoundError extends Error { // translated payloads coming in from another protocol's attempt never carry // `previous_response_id`, so this runs above the fork and never on a wire. export const expandPreviousResponseId = async ( - payload: CanonicalResponsesPayload, - store: StatefulResponsesStore, -): Promise => { + payload: CanonicalOpenAIResponsesPayload, + store: StatefulOpenAIResponsesStore, +): Promise => { const previousResponseId = payload.previous_response_id; if (previousResponseId === undefined || previousResponseId === null) return payload; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index 8c7e272d87..7f1f71a7ca 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -10,7 +10,7 @@ import type { Context } from 'hono'; -import { createResponsesWsSession, type StatefulResponsesStore } from './items/store.ts'; +import { createOpenAIResponsesWsSession, type StatefulOpenAIResponsesStore } from './items/store.ts'; import { openaiResponsesServePipeline, type OpenAIResponsesFacts, type OpenAIResponsesServeExit } from './pipeline.ts'; import { openRunDump } from '../../../dump/run-sink.ts'; import type { TurnDump } from '../../../dump/turn-dump.ts'; @@ -29,11 +29,11 @@ import { SourceStreamState } from '../shared/source-stream-state.ts'; import { move, run } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { RESPONSES_MISSING_TERMINAL_MESSAGE } from '@floway-dev/protocols/openai-responses'; -import { isResponsesTerminalEvent, type CanonicalResponsesPayload, type ClientResponsesStreamEvent, type OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; +import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE } from '@floway-dev/protocols/openai-responses'; +import { isOpenAIResponsesTerminalEvent, type CanonicalOpenAIResponsesPayload, type ClientOpenAIResponsesStreamEvent, type OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; import type { ModelCandidate } from '@floway-dev/provider'; import { toInternalDebugError } from '@floway-dev/provider'; -import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; +import { canonicalizeOpenAIResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; interface WorkerWebSocket extends WebSocket { accept(): void; @@ -70,7 +70,7 @@ type OpenAIResponsesWebSocketUpgradeResolver = ( let _responsesWebSocketUpgradeResolver: OpenAIResponsesWebSocketUpgradeResolver | null = null; -export const initResponsesWebSocketUpgradeResolver = ( +export const initOpenAIResponsesWebSocketUpgradeResolver = ( resolver: OpenAIResponsesWebSocketUpgradeResolver, ): void => { _responsesWebSocketUpgradeResolver = resolver; @@ -99,7 +99,7 @@ export const openaiResponsesWebSocket = async (c: AuthedContext): Promise { +const createOpenAIResponsesWebSocketEvents = (c: AuthedContext): OpenAIResponsesWebSocketHandlers => { // The upgrade authenticates the connection, but every response.create is a // separate data-plane request. Codex deliberately reuses one socket across // turns, so retain the presented credential and resolve it again before each // turn rather than freezing key/user policy at upgrade time. const authenticatedRawKey = apiKeyFromContext(c).key; - const session = createResponsesWsSession(); + const session = createOpenAIResponsesWsSession(); let closed = false; let activeAbortController: AbortController | undefined; let queue = Promise.resolve(); @@ -217,15 +217,15 @@ interface OpenAIResponsesWsTurnFailure { * differ in is the services a chat run is given, which is why it hands those back in the shape * every chat entry hands them over in. */ -const openResponsesWebSocketTurn = ( +const openOpenAIResponsesWebSocketTurn = ( c: AuthedContext, turn: { - readonly payload: CanonicalResponsesPayload; + readonly payload: CanonicalOpenAIResponsesPayload; readonly body: RequestBody; readonly headers: Ingress['headers']; readonly downstreamAbortController: AbortController; readonly backgroundScheduler: BackgroundScheduler; - readonly store: (apiKey: ApiKey, requestStartedAt: number) => StatefulResponsesStore; + readonly store: (apiKey: ApiKey, requestStartedAt: number) => StatefulOpenAIResponsesStore; }, ): ChatPrologue => { // The frame is this turn's request body, so an operator reading the dashboard sees the @@ -268,7 +268,7 @@ const openResponsesWebSocketTurn = ( const handleClientMessage = async ( c: AuthedContext, socket: OpenAIResponsesWebSocketSocket, - session: ReturnType, + session: ReturnType, data: unknown, authenticatedRawKey: string, downstreamAbortController: AbortController, @@ -340,7 +340,7 @@ const handleClientMessage = async ( const payload = openaiResponsesPayloadFromClientSource(source); previousResponseId = payload.previous_response_id ?? undefined; - const prologue = openResponsesWebSocketTurn(c, { + const prologue = openOpenAIResponsesWebSocketTurn(c, { payload, body: requestBody, // The upgrade's own headers are the connection's, and every turn on it is dialled with @@ -359,7 +359,7 @@ const handleClientMessage = async ( openaiResponsesServePipeline(payload, 'events'), move({ 'ingress.http.headers': prologue.headers, - 'ingress.chat.sourceProtocol': 'responses', + 'ingress.chat.sourceProtocol': 'openaiResponses', // A turn on this transport always streams, whatever the client wrote. 'ingress.chat.openaiResponses.wantsStream': true, 'request.chat.openaiResponses': payload, @@ -368,7 +368,7 @@ const handleClientMessage = async ( prologue.services as never, ); - await respondResponsesWebSocket({ socket, eventId, signal, isClosed, prologue, facts, drain, turnFailure }); + await respondOpenAIResponsesWebSocket({ socket, eventId, signal, isClosed, prologue, facts, drain, turnFailure }); } catch (error) { if (signal.aborted || isClosed()) return; if (error instanceof TranslatorInputError) { @@ -417,17 +417,17 @@ const validateClientMessage = (parsed: unknown): OpenAIResponsesWebSocketClientE }; // The transport always streams, whatever the client sent. -const openaiResponsesPayloadFromClientSource = (source: object): CanonicalResponsesPayload => - ({ ...canonicalizeResponsesPayload(source as OpenAIResponsesRequestPayload), stream: true }); +const openaiResponsesPayloadFromClientSource = (source: object): CanonicalOpenAIResponsesPayload => + ({ ...canonicalizeOpenAIResponsesPayload(source as OpenAIResponsesRequestPayload), stream: true }); /** The chain hands its answer up at one key, and this entry assembled it to hand up events * rather than the SSE frames an HTTP body is written from — so what is not a stream at that * key is the one object arm this transport can see. */ const isRenderedStream = ( rendered: OpenAIResponsesFacts['response.chat.openaiResponses.rendered'], -): rendered is AsyncIterable> => Symbol.asyncIterator in rendered; +): rendered is AsyncIterable> => Symbol.asyncIterator in rendered; -const respondResponsesWebSocket = async (input: { +const respondOpenAIResponsesWebSocket = async (input: { readonly socket: OpenAIResponsesWebSocketSocket; readonly eventId: string | undefined; readonly signal: AbortSignal; @@ -458,8 +458,8 @@ const respondResponsesWebSocket = async (input: { const state = new SourceStreamState(); let completion: StreamCompletion = 'error'; try { - let terminalEvent: ClientResponsesStreamEvent | undefined; - const iterator = observeResponsesWebSocketFrames(rendered, state)[Symbol.asyncIterator](); + let terminalEvent: ClientOpenAIResponsesStreamEvent | undefined; + const iterator = observeOpenAIResponsesWebSocketFrames(rendered, state)[Symbol.asyncIterator](); let pendingNext = pendingWsFrameResult(iterator.next()); let completed = false; let stoppedByDownstream = false; @@ -482,7 +482,7 @@ const respondResponsesWebSocket = async (input: { if (next.type === 'keep-alive') { // Extended reasoning turns go completely silent: upstream sends SSE - // `ping` events, `parseResponsesStream` drops them, and no frame at + // `ping` events, `parseOpenAIResponsesStream` drops them, and no frame at // all reaches this socket for minutes. A silent Workers WebSocket // does not reliably survive that. Cloudflare states the // teardown without naming a duration — "when no data is transmitted @@ -580,12 +580,12 @@ const respondResponsesWebSocket = async (input: { // https://github.com/openai/codex/blob/acd540f1581bf30f963fccbcce43ac494102242c/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L792-L799 if (terminalEvent !== undefined) continue; - if (isResponsesTerminalEvent(event)) { + if (isOpenAIResponsesTerminalEvent(event)) { terminalEvent = event; continue; } - if (!sendResponsesEvent(socket, sequence.renumber(event), eventId, ctx.dump)) { + if (!sendOpenAIResponsesEvent(socket, sequence.renumber(event), eventId, ctx.dump)) { stopForDownstream(); return; } @@ -600,12 +600,12 @@ const respondResponsesWebSocket = async (input: { } if (terminalEvent === undefined) { - throw new Error(RESPONSES_MISSING_TERMINAL_MESSAGE); + throw new Error(OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE); } // Renumbered here rather than where it was buffered: keep-alives can still // fire while the generator drains behind the terminal event, and each of // those takes a slot that has to land before the terminal event's own. - if (!sendResponsesEvent(socket, sequence.renumber(terminalEvent), eventId, ctx.dump)) { + if (!sendOpenAIResponsesEvent(socket, sequence.renumber(terminalEvent), eventId, ctx.dump)) { completion = 'cancel'; return; } @@ -646,27 +646,27 @@ const respondResponsesWebSocket = async (input: { /** Reads the turn's own ending off the frames on their way to the socket. The record is not * its business: the edge tees these same frames, so what reaches here is already recorded. */ -const observeResponsesWebSocketFrames = async function* ( - frames: AsyncIterable>, +const observeOpenAIResponsesWebSocketFrames = async function* ( + frames: AsyncIterable>, state: SourceStreamState, -): AsyncGenerator> { +): AsyncGenerator> { for await (const frame of frames) { if (frame.type === 'event') { const event = frame.event; const failed = event.type === 'error' || event.type === 'response.failed'; if (failed) state.failed = true; - if (isResponsesTerminalEvent(event) && !failed) state.completed = true; + if (isOpenAIResponsesTerminalEvent(event) && !failed) state.completed = true; } yield frame; } }; type WsFrameRaceResult = - | { type: 'frame'; result: IteratorResult> } + | { type: 'frame'; result: IteratorResult> } | { type: 'next-error'; error: unknown } | { type: 'keep-alive' }; -const pendingWsFrameResult = (pendingNext: Promise>>): Promise => +const pendingWsFrameResult = (pendingNext: Promise>>): Promise => pendingNext.then( (result): WsFrameRaceResult => ({ type: 'frame', result }), (error): WsFrameRaceResult => ({ type: 'next-error', error }), @@ -693,7 +693,7 @@ const nextFrameOrKeepAlive = async (pendingFrame: Promise): P }; interface DownstreamSequence { - renumber(event: ClientResponsesStreamEvent): ClientResponsesStreamEvent; + renumber(event: ClientOpenAIResponsesStreamEvent): ClientOpenAIResponsesStreamEvent; take(): number; } @@ -780,9 +780,9 @@ const sendError = ( // envelope is not a streaming event, and the keep-alive is a slug-prefixed // extension the union deliberately does not model. Both keep the untyped // `sendJson`. -const sendResponsesEvent = ( +const sendOpenAIResponsesEvent = ( socket: OpenAIResponsesWebSocketSocket, - event: ClientResponsesStreamEvent, + event: ClientOpenAIResponsesStreamEvent, eventId?: string, dump?: TurnDump | null, ): boolean => sendJson(socket, event, eventId, dump); diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts index 1094281bfc..5487c02ab7 100644 --- a/packages/gateway/src/data-plane/chat/prologue.ts +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -13,7 +13,7 @@ import type { ChatServices } from './stages.ts'; import type { ApiKey } from '../../repo/types.ts'; import type { AttemptSelector } from '../pipeline/facts.ts'; import { gatewayCtxOptions, prologueFor, runDumpOf, type Ingress, type Prologue } from '../pipeline/serve.ts'; -import type { StatefulResponsesStore } from './openai-responses/items/store.ts'; +import type { StatefulOpenAIResponsesStore } from './openai-responses/items/store.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import type { ModelCandidate } from '@floway-dev/provider'; @@ -30,7 +30,7 @@ export const openChatPrologue = ( readonly model?: string; /** Native Responses entries persist their items; every other source gets a scratchpad, * so the server-tool shim's request-private state always has a home. */ - readonly storeFactory: (apiKey: ApiKey, requestStartedAt: number) => StatefulResponsesStore; + readonly storeFactory: (apiKey: ApiKey, requestStartedAt: number) => StatefulOpenAIResponsesStore; }, ): ChatPrologue => { // One options object, read twice: the context is built from it, and the run recording it diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts index 713f10e1d0..3467971f90 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts @@ -15,7 +15,7 @@ import { withInteractionIdHeaderSet } from './set-interaction-id-header.ts'; import { withVisionHeaderSet } from './set-vision-header.ts'; import { withCacheControlExtensionsStripped } from './strip-cache-control-extensions.ts'; import { withEagerInputStreamingStripped } from './strip-eager-input-streaming.ts'; -import type { CopilotMessagesBoundaryInterceptor, CopilotMessagesCountTokensBoundaryInterceptor } from './types.ts'; +import type { CopilotAnthropicMessagesBoundaryInterceptor, CopilotAnthropicMessagesCountTokensBoundaryInterceptor } from './types.ts'; // Order rationale, split into two lanes that run back-to-back: // @@ -35,7 +35,7 @@ import type { CopilotMessagesBoundaryInterceptor, CopilotMessagesCountTokensBoun // outgoing payload; withTopLevelCacheControlApplied runs before // withCacheControlExtensionsStripped so the ported marker on the last // cacheable block is cleaned in the same pass. withSpeedFast strips the -// client's `speed` field (already consumed by callMessages for raw-variant +// client's `speed` field (already consumed by callAnthropicMessages for raw-variant // selection) and post-`run()` stamps `usage.speed='fast'` onto outbound // message_start/message_delta events when Fast Mode was requested. The // header lane closes by normalizing admitted caller beta intent, then adding @@ -45,13 +45,13 @@ import type { CopilotMessagesBoundaryInterceptor, CopilotMessagesCountTokensBoun // compact-tagged value above — that mirrors the pre-boundary target-side // override. // -// `withMessagesWebSearchShim` is intentionally NOT registered here. The shim +// `withAnthropicMessagesWebSearchShim` is intentionally NOT registered here. The shim // is the gateway's rather than this provider's — it rewrites Anthropic's // native server tool into an ordinary client tool and runs the search itself, // which is work no provider boundary does. This provider opts into it by // listing `messages-web-search-shim` in its default flag set (see // COPILOT_DEFAULT_FLAGS in ../../defaults.ts). -export const COPILOT_MESSAGES_BOUNDARY = [ +export const COPILOT_ANTHROPIC_MESSAGES_BOUNDARY = [ rewriteContextWindowError, withCompactHeadersSet, withClaudeAgentHeadersSet, @@ -65,7 +65,7 @@ export const COPILOT_MESSAGES_BOUNDARY = [ withVisionHeaderSet, withInitiatorHeaderSet, withAnthropicBetaNormalized, -] as const satisfies readonly CopilotMessagesBoundaryInterceptor[]; +] as const satisfies readonly CopilotAnthropicMessagesBoundaryInterceptor[]; // /v1/messages/count_tokens is a one-shot HTTP exchange that returns the raw // upstream Response. The Copilot provider applies vision detection, @@ -79,9 +79,9 @@ export const COPILOT_MESSAGES_BOUNDARY = [ // raw Response at all, and the remaining payload mutators and header setters // each answer something we observed on the generation endpoint, with no // equivalent need seen on count_tokens. -export const COPILOT_MESSAGES_COUNT_TOKENS_BOUNDARY = [ +export const COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY = [ withInlineImagesCompressed, withVisionHeaderSet, withInitiatorHeaderSet, withAnthropicBetaNormalized, -] as const satisfies readonly CopilotMessagesCountTokensBoundaryInterceptor[]; +] as const satisfies readonly CopilotAnthropicMessagesCountTokensBoundaryInterceptor[]; From 8aefef59ab54bae2deef7d8b8e9c825fc0966901 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 05:56:56 +0800 Subject: [PATCH 080/156] test(chat): follow the rename into the assertions that read a name The affinity carrier domains are vendor-qualified on the wire the codec seals with, the composition error quotes the fact key it refused, and two Responses validation sentences name the vendor. The chat suites were still asserting the pre-rename spellings. --- .../data-plane/chat/chatcompletions-pipeline_test.ts | 2 +- .../gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts | 2 +- .../__tests__/data-plane/chat/messages-pipeline_test.ts | 2 +- .../data-plane/chat/openai-responses/websocket_test.ts | 4 ++-- .../__tests__/data-plane/chat/responses-compact_test.ts | 2 +- .../__tests__/data-plane/chat/responses-pipeline_test.ts | 4 ++-- .../__tests__/data-plane/chat/translated-wires_test.ts | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts index e583096bec..a107dc2631 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts @@ -201,7 +201,7 @@ describe('the chat completions chain', () => { expect(carried).toHaveLength(1); // It names the upstream that answered, sealed under this run's own secret — which is what // lets the next turn be pinned to it. - expect(await gateway.affinity.codec.unwrap(carried[0]!, 'chat-completions.reasoning_opaque')).toMatchObject({ + expect(await gateway.affinity.codec.unwrap(carried[0]!, 'openai-chat-completions.reasoning_opaque')).toMatchObject({ kind: 'owned', affinity: { upstreamId: 'up_a', modelId: 'chat-model' }, }); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts index bbd5b4bbe6..5ac8629de3 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts @@ -285,7 +285,7 @@ describe('the gemini pipeline', () => { expect(carried).toHaveLength(1); // It names the upstream that answered, sealed under this run's own secret — which is what // lets the next turn be pinned to it. - expect(await gateway.affinity.codec.unwrap(carried[0]!, 'gemini.part.thoughtSignature')).toMatchObject({ + expect(await gateway.affinity.codec.unwrap(carried[0]!, 'gemini-generate-content.part.thoughtSignature')).toMatchObject({ kind: 'owned', affinity: { upstreamId: 'up_a', modelId: 'gemini-2.5-pro' }, }); diff --git a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts index c8a524d75b..1b258c42a0 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts @@ -236,7 +236,7 @@ describe('the messages chain', () => { expect(carrier).toHaveLength(1); // It names the upstream that answered, sealed under this run's own secret — which is what // lets the next turn be pinned to it. - expect(await gateway.affinity.codec.unwrap(carrier[0]!.content_block!.data, 'messages.redacted_thinking.data')).toMatchObject({ + expect(await gateway.affinity.codec.unwrap(carrier[0]!.content_block!.data, 'anthropic-messages.redacted_thinking.data')).toMatchObject({ kind: 'owned', affinity: { upstreamId: 'up_a', modelId: 'claude-model' }, }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index de6889d046..fbc8cbff8b 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -694,7 +694,7 @@ test('Responses WebSocket returns invalid_request_error for malformed client mes assertEquals(invalidInputMessage.error, { type: 'invalid_request_error', code: 'invalid_request_error', - message: 'Responses input must be a string or an array.', + message: 'OpenAI Responses input must be a string or an array.', param: 'input', }); @@ -712,7 +712,7 @@ test('Responses WebSocket returns invalid_request_error for malformed client mes error: { type: 'invalid_request_error', code: 'invalid_request_error', - message: 'Untyped Responses input items require a valid role and content.', + message: 'Untyped OpenAI Responses input items require a valid role and content.', param: 'input[0]', }, }]); diff --git a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts index 260b9ef393..ec18ec07dc 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts @@ -186,7 +186,7 @@ const rendered = (facts: Record): Record => * own state is sealed around it on the way out — that is what pins the next turn to the * upstream that issued this one — so the carrier is opened before the blob is read. */ const summaryIn = async (encryptedContent: string): Promise => { - const carrier = await gateway.affinity.codec.unwrap(encryptedContent, 'responses.compaction.encrypted_content'); + const carrier = await gateway.affinity.codec.unwrap(encryptedContent, 'openai-responses.compaction.encrypted_content'); if (carrier.kind !== 'owned' || carrier.value === undefined) throw new Error('expected the turn-s own carrier around the blob'); const items = decodeBase64UrlJson(carrier.value) as { content: { text: string }[] }[] | null; if (items === null) throw new Error('expected a shim-encoded compaction blob'); diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts index 339f329f97..1b002f4241 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts @@ -234,7 +234,7 @@ const summaryIn = async ( gateway: ReturnType, encryptedContent: string, ): Promise => { - const carrier = await gateway.affinity.codec.unwrap(encryptedContent, 'responses.compaction.encrypted_content'); + const carrier = await gateway.affinity.codec.unwrap(encryptedContent, 'openai-responses.compaction.encrypted_content'); if (carrier.kind !== 'owned' || carrier.value === undefined) throw new Error('expected the turn-s own carrier around the blob'); const items = decodeBase64UrlJson(carrier.value) as { content: { text: string }[] }[] | null; if (items === null) throw new Error('expected a shim-encoded compaction blob'); @@ -368,7 +368,7 @@ describe('the responses chain', () => { // It names the upstream that answered, sealed under this run's own secret — which is what // lets the next turn be pinned to it. The item is ours rather than the upstream's, and it // says so, so a turn quoting it back is not read as reasoning the upstream produced. - expect(await gateway.affinity.codec.unwrap(carrier[0]!.encrypted_content!, 'responses.reasoning.encrypted_content')).toMatchObject({ + expect(await gateway.affinity.codec.unwrap(carrier[0]!.encrypted_content!, 'openai-responses.reasoning.encrypted_content')).toMatchObject({ kind: 'owned', syntheticItem: true, affinity: { upstreamId: 'up_a', modelId: 'responses-model' }, diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 3e434db4d7..00018416b0 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -547,7 +547,7 @@ describe('a rule that speaks about one protocol-s wire', () => { }); expect(() => compose('leaving', [leaving, ...anthropicMessagesWire('r')])) - .toThrow(/needs request\.chat\.messages, which handOff:.* consumed above it/); + .toThrow(/needs request\.chat\.anthropicMessages, which handOff:.* consumed above it/); expect(() => compose('arriving', [arriving, ...anthropicMessagesWire('r')])).not.toThrow(); }); }); From fbddee10cafed8f47b8bd6eb7675be4f5d415bbb Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 06:07:38 +0800 Subject: [PATCH 081/156] refactor(chat): say the protocol in full in prose, test names and file names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chat chains still read as if only one vendor spoke each shape. Comments, doc blocks, test names and the two client-facing sentences the WebSocket entry writes now spell "OpenAI Chat Completions", "Anthropic Messages", "OpenAI Responses" and "Gemini generateContent", and the eleven flat chat test files take the kebab spelling their non-chat siblings already use. The longer names break the article before them, so "a OpenAI" and "a Anthropic" are rewritten wherever they fell out — including two the earlier rename left, in `server-tool-shim.ts` and in the Anthropic-Messages-via-OpenAI-Responses events. Vendor wording stays: "Unknown Gemini model action" is what a client reads, "OpenAI's Responses API" and "Gemini's `error` channel" are the vendors' own names for their surfaces. --- .../data-plane/chat-interceptors_test.ts | 20 ++++----- ...> anthropic-messages-count-tokens_test.ts} | 4 +- ...ts => anthropic-messages-pipeline_test.ts} | 4 +- ...ini-generate-content-count-tokens_test.ts} | 14 +++--- ... gemini-generate-content-pipeline_test.ts} | 18 ++++---- ... openai-chat-completions-pipeline_test.ts} | 2 +- ...st.ts => openai-responses-compact_test.ts} | 8 ++-- ...t.ts => openai-responses-pipeline_test.ts} | 8 ++-- .../chat/openai-responses/errors_test.ts | 4 +- .../chat/openai-responses/websocket_test.ts | 44 +++++++++---------- ...t.ts => routes-anthropic-messages_test.ts} | 12 ++--- ...=> routes-gemini-generate-content_test.ts} | 29 ++++++------ ...=> routes-openai-chat-completions_test.ts} | 8 ++-- ...est.ts => routes-openai-responses_test.ts} | 2 +- .../data-plane/chat/translated-wires_test.ts | 39 ++++++++-------- .../chat/translation-refusal_test.ts | 10 ++--- .../data-plane/pipeline/assembly_test.ts | 2 +- .../chat/anthropic-messages/count-tokens.ts | 20 ++++----- .../chat/anthropic-messages/errors.ts | 2 +- .../chat/anthropic-messages/http.ts | 4 +- .../interceptors/answer-claude-code-probe.ts | 2 +- .../interceptors/web-search-shim.ts | 2 +- .../chat/anthropic-messages/pipeline.ts | 26 +++++------ packages/gateway/src/data-plane/chat/facts.ts | 2 +- .../gemini-generate-content/count-tokens.ts | 18 ++++---- .../chat/gemini-generate-content/errors.ts | 2 +- .../chat/gemini-generate-content/http.ts | 4 +- .../chat/gemini-generate-content/pipeline.ts | 43 +++++++++--------- .../gateway/src/data-plane/chat/handoff.ts | 2 +- .../src/data-plane/chat/interceptors.ts | 40 ++++++++--------- .../chat/openai-chat-completions/pipeline.ts | 18 ++++---- .../chat/openai-responses/client-output.ts | 2 +- .../chat/openai-responses/compact-shim.ts | 2 +- .../chat/openai-responses/compact.ts | 12 ++--- .../interceptors/server-tool-shim.ts | 2 +- .../chat/openai-responses/items/store.ts | 9 ++-- .../chat/openai-responses/pipeline.ts | 34 +++++++------- .../chat/openai-responses/websocket.ts | 4 +- .../gateway/src/data-plane/chat/prologue.ts | 2 +- .../chat/shared/source-stream-state.ts | 2 +- .../gateway/src/data-plane/chat/stages.ts | 2 +- .../events.ts | 2 +- 42 files changed, 246 insertions(+), 240 deletions(-) rename packages/gateway/__tests__/data-plane/chat/{messages-count-tokens_test.ts => anthropic-messages-count-tokens_test.ts} (98%) rename packages/gateway/__tests__/data-plane/chat/{messages-pipeline_test.ts => anthropic-messages-pipeline_test.ts} (98%) rename packages/gateway/__tests__/data-plane/chat/{gemini-count-tokens_test.ts => gemini-generate-content-count-tokens_test.ts} (93%) rename packages/gateway/__tests__/data-plane/chat/{gemini-pipeline_test.ts => gemini-generate-content-pipeline_test.ts} (95%) rename packages/gateway/__tests__/data-plane/chat/{chatcompletions-pipeline_test.ts => openai-chat-completions-pipeline_test.ts} (99%) rename packages/gateway/__tests__/data-plane/chat/{responses-compact_test.ts => openai-responses-compact_test.ts} (97%) rename packages/gateway/__tests__/data-plane/chat/{responses-pipeline_test.ts => openai-responses-pipeline_test.ts} (98%) rename packages/gateway/__tests__/data-plane/chat/{routes-messages_test.ts => routes-anthropic-messages_test.ts} (93%) rename packages/gateway/__tests__/data-plane/chat/{routes-gemini_test.ts => routes-gemini-generate-content_test.ts} (88%) rename packages/gateway/__tests__/data-plane/chat/{routes-chat-completions_test.ts => routes-openai-chat-completions_test.ts} (95%) rename packages/gateway/__tests__/data-plane/chat/{responses-routes_test.ts => routes-openai-responses_test.ts} (97%) diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index a0a39a33c7..7c85d4eaef 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -120,7 +120,7 @@ const anthropicMessagesPayload = (messages: unknown[], rest: Record = {}): OpenAIResponsesPayload => ({ model: 'm', input, ...rest }) as unknown as OpenAIResponsesPayload; -describe('the Chat Completions interceptors, as stages', () => { +describe('the OpenAI Chat Completions interceptors, as stages', () => { it('rewrites the roles its flags name, in the settled order', async () => { const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ @@ -426,7 +426,7 @@ const assistantReplay = (reasoning: Record): OpenAIChatCompleti const assistantOf = (payload: unknown): Record => (payload as OpenAIChatCompletionsPayload).messages[1] as unknown as Record; -describe('the Chat Completions vendor dialects, as stages', () => { +describe('the OpenAI Chat Completions vendor dialects, as stages', () => { it('projects the canonical assistant reasoning onto the one field DeepSeek reads', async () => { const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload(assistantReplay({ @@ -650,7 +650,7 @@ describe('the Chat Completions vendor dialects, as stages', () => { }); }); -describe('the Messages interceptors, as stages', () => { +describe('the Anthropic Messages interceptors, as stages', () => { it('turns every inline system message into a user one', async () => { const { down } = await runStage(applyRoleCompatibilityToAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': anthropicMessagesPayload([ @@ -751,7 +751,7 @@ describe('the Messages interceptors, as stages', () => { }); }); -describe('the Gemini interceptors, as stages', () => { +describe('the Gemini generateContent interceptors, as stages', () => { it('drops the part fields no translation carries, and a part they were all of', async () => { const { down } = await runStage(stripUnsupportedPartFieldsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': { @@ -859,7 +859,7 @@ describe('the Gemini interceptors, as stages', () => { }); }); -describe('the Responses interceptors, as stages', () => { +describe('the OpenAI Responses interceptors, as stages', () => { const rolesOf = (down: Record): unknown[] => ((down['request.chat.openaiResponses'] as OpenAIResponsesPayload).input as OpenAIResponsesInputItem[]) .map(item => (item as { role?: unknown }).role); @@ -1035,7 +1035,7 @@ const usageOf = (frame: ProtocolFrame): Record { - it('assembles and runs the Messages array', async () => { + it('assembles and runs the Anthropic Messages array', async () => { const { down } = await runChain('response.chat.anthropicMessages', [ stripBillingAttributionFromAnthropicMessages, disableReasoningOnForcedToolChoiceForAnthropicMessages, @@ -1057,7 +1057,7 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => expect(after.messages.map(message => message.role)).toEqual(['user']); }); - it('assembles and runs the Gemini array', async () => { + it('assembles and runs the Gemini generateContent array', async () => { const answer = streamOf([ eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }, { text: 'the answer' }] }, finishReason: 'STOP' }] }), ]); @@ -1085,7 +1085,7 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => // The one ordering between stages rather than inside one: the sentinel is the gateway's // canonical form, and a vendor normalizer can only put it on the wire in the vendor's shape // because it runs after the stage that wrote it. - it('assembles and runs the Responses array, the vendor normalizers last', async () => { + it('assembles and runs the OpenAI Responses array, the vendor normalizers last', async () => { const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); const { down, up } = await runChain('response.chat.openaiResponses', [ disableReasoningOnForcedToolChoiceForOpenAIResponses, @@ -1117,12 +1117,12 @@ describe('a family\'s interceptor array, in the order its chain runs it', () => expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 150 }); }); - // The Chat Completions wire's own array, which is not one family's: every source protocol + // The OpenAI Chat Completions wire's own array, which is not one family's: every source protocol // that reaches an upstream over this endpoint runs it. What the order states is one thing in // each direction. Going down, the usage chunk is asked for before any vendor can rewrite the // body. Coming back, the vendor dialect has the first say — so the fold reads a cache count // under OpenAI's name rather than DeepSeek's, and the carrier split reads the folded total. - it('assembles and runs the Chat Completions wire, the vendor dialects innermost', async () => { + it('assembles and runs the OpenAI Chat Completions wire, the vendor dialects innermost', async () => { const answer = streamOf([eventFrame({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], diff --git a/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts similarity index 98% rename from packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts rename to packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts index f19bbc5424..09416c883d 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts @@ -1,4 +1,4 @@ -// The Messages count-tokens chain, run. What is written down here is what only running it +// The Anthropic Messages count-tokens chain, run. What is written down here is what only running it // can say: that a measurement is taken on the body generation would have sent — the payload // affinity materialized, the alias' own rules, the three request rules and the web-search // rewrite — that a refusal is answered in the upstream's own words after every candidate has @@ -310,7 +310,7 @@ describe('the messages count-tokens chain', () => { expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ input_tokens: 7 }); }); - // Only a native Messages endpoint measures — no translation carries the question — so a + // Only a native Anthropic Messages endpoint measures — no translation carries the question — so a // candidate that would serve generation over a translated wire cannot serve this, and the // refusal names the endpoint the client actually addressed. it('refuses a candidate no wire can measure on, naming the count endpoint', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts similarity index 98% rename from packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts rename to packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts index 1b258c42a0..ea61beafdf 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts @@ -1,4 +1,4 @@ -// The Messages chain, run. Assembly is checked where every family's is; what is written +// The Anthropic Messages chain, run. Assembly is checked where every family's is; what is written // down here is what only running it can say — that the edge writes Anthropic's own named // SSE events when the client asked to stream and reassembles them into one message when it // did not, that it hands the turn's own state back for the client to carry, that a refusal @@ -140,7 +140,7 @@ const collect = async (rendered: unknown): Promise => { }; // The carrier the edge writes back is a redacted_thinking block holding this turn's own -// state. Messages has real block boundaries, so it takes content block 0 of its own: its +// state. Anthropic Messages has real block boundaries, so it takes content block 0 of its own: its // start and stop are the two extra frames, and the turn's own blocks answer behind it. /** The turn's own frames, with the carrier block's dropped. */ diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts similarity index 93% rename from packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts rename to packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts index 643b620c0f..f3b8768188 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts @@ -1,5 +1,5 @@ -// The Gemini count-tokens chain, run. What is written down here is what only running it can -// say: that the question is asked in Messages and the answer read back out as Google's, that +// The Gemini generateContent count-tokens chain, run. What is written down here is what only running it can +// say: that the question is asked in Anthropic Messages and the answer read back out as Google's, that // what no translation can carry is gone before the question is asked, that a refusal keeps // the upstream's own status, and that measuring bills nothing. @@ -107,9 +107,9 @@ beforeEach(() => { }); describe('the gemini count-tokens chain', () => { - // Gemini has no endpoint that answers this, so the question is asked in Messages and the + // Gemini generateContent has no endpoint that answers this, so the question is asked in Anthropic Messages and the // counts are read back out under the name this protocol gives them. - it('asks the question in Messages and answers in Google-s own envelope', async () => { + it('asks the question in Anthropic Messages and answers in Google-s own envelope', async () => { let sent: AnthropicMessagesPayload | undefined; let seenModel: { readonly id: string } | undefined; resolves([candidate(async (model, body) => { @@ -158,7 +158,7 @@ describe('the gemini count-tokens chain', () => { }); }); - // Gemini has no wire of its own, so what no translation can carry cannot be sent whichever + // Gemini generateContent has no wire of its own, so what no translation can carry cannot be sent whichever // candidate answers — and a count taken before the strip would measure a request the // translator would refuse. it('strips what no translation carries before asking', async () => { @@ -179,7 +179,7 @@ describe('the gemini count-tokens chain', () => { }); // Anthropic's beta flags are the client's own only when the client spoke that protocol, and - // a Gemini turn asked for nothing on that wire. + // a Gemini generateContent turn asked for nothing on that wire. it('does not carry another protocol-s beta flags across the boundary', async () => { let seen: AnthropicMessagesUpstreamCallOptions | undefined; resolves([candidate(async (_model, _body, _signal, opts) => { seen = opts; return answered({ input_tokens: 1 }); })]); @@ -202,7 +202,7 @@ describe('the gemini count-tokens chain', () => { }); }); - // Only an upstream's own Messages endpoint measures, so a candidate that would serve + // Only an upstream's own Anthropic Messages endpoint measures, so a candidate that would serve // generation over a translated wire cannot serve this. it('refuses a candidate no wire can measure on, naming the action', async () => { resolves([candidate(async () => answered({ input_tokens: 0 }), { endpoints: { openaiChatCompletions: {} } })]); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts similarity index 95% rename from packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts rename to packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts index 5ac8629de3..4fc6a8bf18 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts @@ -1,6 +1,6 @@ -// Gemini's pipeline, assembled and run. Gemini is the source-only family and the one with no +// Gemini generateContent's pipeline, assembled and run. Gemini generateContent is the source-only family and the one with no // wire of its own, so what a run of it has to show beyond the entry contract is the round -// trip: a turn translated out to Chat Completions, an answer translated back, the turn's own +// trip: a turn translated out to OpenAI Chat Completions, an answer translated back, the turn's own // state handed back on a Part for the client to carry, and the reading taken on the dialect // the upstream actually spoke rather than on the one the client did. @@ -82,7 +82,7 @@ const turn = (): AsyncIterable> }); /** The wire below closed cleanly and never finished the turn: no choice ever carried a - * finish reason, so nothing that comes out of the translation is a Gemini terminal event. */ + * finish reason, so nothing that comes out of the translation is a Gemini generateContent terminal event. */ const unfinishedTurn = (): AsyncIterable> => ({ [Symbol.asyncIterator]: () => (async function* () { yield chunk({ content: 'he' }); @@ -165,7 +165,7 @@ const collect = async (rendered: unknown): Promise => { }; /** The turn's own answer. The carrier the edge writes back rides as a `thoughtSignature` on - * a Part rather than as an event of its own, so taking it off leaves what Gemini itself + * a Part rather than as an event of its own, so taking it off leaves what Gemini generateContent itself * said. */ const withoutCarrier = (event: GeminiGenerateContentResult): GeminiGenerateContentResult => ({ ...event, @@ -200,7 +200,7 @@ describe('the gemini pipeline', () => { ]); }); - it('translates the turn out, dials the Chat Completions wire, and streams Gemini frames back', async () => { + it('translates the turn out, dials the OpenAI Chat Completions wire, and streams Gemini generateContent frames back', async () => { let sent: Record | undefined; resolves([candidate('up_a', async (_model, body) => { sent = body as Record; @@ -226,7 +226,7 @@ describe('the gemini pipeline', () => { await drain(); }); - // Gemini carries its state in `thoughtSignature` parts, which only mean anything to the + // Gemini generateContent carries its state in `thoughtSignature` parts, which only mean anything to the // upstream that issued them — so the turn that is translated is the one affinity rewrote // for this candidate, and the candidate that answered is marked for the next turn. That // turn enters the record below the fork because everything between there and the dial @@ -245,7 +245,7 @@ describe('the gemini pipeline', () => { await collect(facts['response.chat.geminiGenerateContent.rendered']); // The handoff consumed `request.chat.geminiGenerateContent` and provided the wire's own key, so what the - // record still carries below the fork is the translated turn and not the Gemini one — the + // record still carries below the fork is the translated turn and not the Gemini generateContent one — the // whole of what a translation is, said in the fact space. expect((facts as Record)['request.chat.geminiGenerateContent']).toBeUndefined(); expect((facts as Record)['request.chat.openaiChatCompletions']) @@ -256,7 +256,7 @@ describe('the gemini pipeline', () => { await drain(); }); - it('folds the frames into one Gemini result for a client that did not ask to stream', async () => { + it('folds the frames into one Gemini generateContent result for a client that did not ask to stream', async () => { resolves([candidate('up_a', async () => streamed(turn()))]); const { facts, drain } = await serve(entryFacts({ 'ingress.chat.geminiGenerateContent.wantsStream': false })); @@ -310,7 +310,7 @@ describe('the gemini pipeline', () => { expect(tried).toEqual(['up_a', 'up_b']); // Every candidate failed, so the last failure is the base: the client sees the status the // last upstream actually returned, and its words. The shape around them is this protocol's, - // because every Gemini wire is a translation and the object that came back was written by + // because every Gemini generateContent wire is a translation and the object that came back was written by // whatever protocol the candidate was dialled on. expect(facts['response.http.status']).toBe(400); const rendered = facts['response.chat.geminiGenerateContent.rendered'] as { error: { code: number; message: string; status: string } }; diff --git a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts similarity index 99% rename from packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts index a107dc2631..c3dfc6e74c 100644 --- a/packages/gateway/__tests__/data-plane/chat/chatcompletions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts @@ -1,4 +1,4 @@ -// The Chat Completions chain, run. Assembly is checked where every family's is; what is +// The OpenAI Chat Completions chain, run. Assembly is checked where every family's is; what is // written down here is what only running it can say — that the edge folds a stream into one // object when the client did not ask to stream, that it hands the turn's own state back for // the client to carry, that a refusal keeps the upstream's own status and words, and that a diff --git a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts similarity index 97% rename from packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts index ec18ec07dc..a925a06ed5 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts @@ -1,4 +1,4 @@ -// The Responses compaction chain, run. Assembly is checked where every family's is; what is +// The OpenAI Responses compaction chain, run. Assembly is checked where every family's is; what is // written down here is what only running it can say — that an upstream whose own endpoint // compacts is asked to and answered under an id this gateway minted, that one with no // compaction wire is sent the compactor's turn over the wires generation would have used and @@ -84,7 +84,7 @@ const compacts = ( return { action: 'compact', ok: true, result: compaction(overrides), modelKey: 'responses-model-key' }; }; -/** A generate turn over the Responses wire that produces one assistant message. */ +/** A generate turn over the OpenAI Responses wire that produces one assistant message. */ const generates = (text: string, seen: { body?: Record; action?: unknown } = {}) => async (_model: unknown, body: unknown, action: unknown): Promise => { seen.body = body as Record; @@ -111,7 +111,7 @@ const generates = (text: string, seen: { body?: Record; action? }; }; -/** The same turn over the Messages wire, which is the only way a Messages-only candidate can +/** The same turn over the Anthropic Messages wire, which is the only way an Anthropic Messages-only candidate can * be reached: the compaction crosses into that protocol and its frames come back translated. */ const anthropicMessagesTurn = (text: string, seen: { body?: Record } = {}) => async (_model: unknown, body: unknown): Promise> => { @@ -239,7 +239,7 @@ describe('the responses compaction chain', () => { expect(seen.body).not.toHaveProperty('model'); }); - // No translation carries a compaction, so a Messages candidate has none to dial. The turn + // No translation carries a compaction, so an Anthropic Messages candidate has none to dial. The turn // it is sent instead is an ordinary generate turn against the compactor's prompt, over the // wire generation would have used — which is the whole of what the action pivot was. it('simulates a compaction over a candidate with no compaction wire', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts similarity index 98% rename from packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts index 1b002f4241..16b31d1077 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -1,4 +1,4 @@ -// The Responses chain, run. Assembly is checked where every family's is; what is written +// The OpenAI Responses chain, run. Assembly is checked where every family's is; what is written // down here is what only running it can say — that the edge terminates the client's stream // on `[DONE]` however the upstream's ended, that it folds a stream into one response object // when the client did not ask to stream, that it hands the turn's own state back for the @@ -107,7 +107,7 @@ const stream = (...events: readonly OpenAIResponsesStreamEvent[]): ProviderOpenA })(), }); -/** One assistant message over the Messages wire, which is the only way a Messages-only +/** One assistant message over the Anthropic Messages wire, which is the only way an Anthropic Messages-only * candidate can be reached: the turn crosses into that protocol and comes back translated. */ const anthropicMessagesTurn = (text: string, seen: { body?: Record } = {}) => async (_model: unknown, body: unknown): Promise> => { @@ -540,7 +540,7 @@ describe('the responses chain', () => { }); // A provider owns the body it is dialled with and shapes it in place, down to nested nodes — - // Copilot marks individual items for caching, and its Messages boundary writes into a nested + // Copilot marks individual items for caching, and its Anthropic Messages boundary writes into a nested // field — while the record that body is built from is deep-frozen. A dial that handed over a // shallow copy satisfies every type in the system and throws at the first nested write, // answering the client with a 502 raised where nothing can explain it. @@ -643,7 +643,7 @@ describe('the responses chain', () => { // No translation carries a compaction, and neither translator models the item that asks for // one — so on a candidate this protocol can only reach through one, the shim is structurally // required rather than opted into, and the flag has nothing to say. - it('answers a turn that asked for a compaction over a candidate with no Responses wire', async () => { + it('answers a turn that asked for a compaction over a candidate with no OpenAI Responses wire', async () => { const seen: { body?: Record } = {}; affinityPayload = asksForCompaction; resolves([candidate( diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts index a8a388d810..89b4591d37 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts @@ -1,4 +1,4 @@ -// The Responses-only refusal, through the same throw/catch. Its shape is the reason the +// The OpenAI Responses-only refusal, through the same throw/catch. Its shape is the reason the // mechanism is generic over the failure type: the store finds the missing item deep inside a // walk, and the stage that started the walk is what answers. @@ -8,7 +8,7 @@ import type { OpenAIResponsesServeFailure } from '../../../../src/data-plane/cha import { throwChatServeFailure, tryCatchChatServeFailure } from '../../../../src/data-plane/chat/shared/errors.ts'; import { assertEquals, assertThrows } from '@floway-dev/test-utils'; -test('round-trips the Responses-only item-not-found failure through throw/catch', () => { +test('round-trips the OpenAI Responses-only item-not-found failure through throw/catch', () => { const failure: OpenAIResponsesServeFailure = { kind: 'item-not-found', itemId: 'msg_abc' }; const error = assertThrows(() => throwChatServeFailure(failure)); assertEquals(tryCatchChatServeFailure(error), failure); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index fbc8cbff8b..878f80bcf1 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -149,7 +149,7 @@ const completeOpenAIResponsesTurn = async ( await waitForMicrotasks(); }; -test('Responses WebSocket forwards stream events, echoes event_id, and ends the turn on the terminal event', async () => { +test('OpenAI Responses WebSocket forwards stream events, echoes event_id, and ends the turn on the terminal event', async () => { const { apiKey } = await setupAppTest(); await withMockedFetch( async request => { @@ -211,7 +211,7 @@ test('Responses WebSocket forwards stream events, echoes event_id, and ends the ); }); -test('Responses WebSocket starts capturing on the next turn when dump retention is enabled after upgrade', async () => { +test('OpenAI Responses WebSocket starts capturing on the next turn when dump retention is enabled after upgrade', async () => { const { apiKey, repo } = await setupAppTest(); const dumps = installDumpStubs(initDumpStore, initDumpBroker); @@ -251,7 +251,7 @@ test('Responses WebSocket starts capturing on the next turn when dump retention ); }); -test('Responses WebSocket stops capturing on the next turn when dump retention is disabled after upgrade', async () => { +test('OpenAI Responses WebSocket stops capturing on the next turn when dump retention is disabled after upgrade', async () => { const { apiKey, repo } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); @@ -271,7 +271,7 @@ test('Responses WebSocket stops capturing on the next turn when dump retention i ); }); -test('Responses WebSocket dump responseBytes equals the UTF-8 payload bytes sent downstream', async () => { +test('OpenAI Responses WebSocket dump responseBytes equals the UTF-8 payload bytes sent downstream', async () => { const { apiKey, repo } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); @@ -311,7 +311,7 @@ const recordedFrameCount = (record: Parameters[0]): number = // into the record, and this transport, which reads it to write each event to the socket — so a // second tee is invisible everywhere except here, and it would make the collected view the // dashboard replays show the whole turn twice. -test('Responses WebSocket records one frame per event it sent', async () => { +test('OpenAI Responses WebSocket records one frame per event it sent', async () => { const { apiKey, repo } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); @@ -344,7 +344,7 @@ test('Responses WebSocket records one frame per event it sent', async () => { // after the last frame has gone out. The rows are filtered to this turn's own key because // every `setupAppTest` mints a fresh one, and background writes from earlier files in the // suite resolve against whichever repo was installed last. -test('Responses WebSocket settles a streamed turn once, with what the chain read off it', async () => { +test('OpenAI Responses WebSocket settles a streamed turn once, with what the chain read off it', async () => { const { apiKey, repo } = await setupAppTest(); const dumps = installDumpStubs(initDumpStore, initDumpBroker); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); @@ -377,7 +377,7 @@ test('Responses WebSocket settles a streamed turn once, with what the chain read assertEquals(meta.outputTokens, 5); }); -test('Responses WebSocket rejects the next turn after its API key is rotated', async () => { +test('OpenAI Responses WebSocket rejects the next turn after its API key is rotated', async () => { const { apiKey, repo } = await setupAppTest(); await withSuccessfulOpenAIResponsesUpstream( @@ -409,7 +409,7 @@ test('Responses WebSocket rejects the next turn after its API key is rotated', a ); }); -test('Responses WebSocket reports a failed turn when an output item cannot be persisted', async () => { +test('OpenAI Responses WebSocket reports a failed turn when an output item cannot be persisted', async () => { const { apiKey, repo } = await setupAppTest(); const persistence = vi.spyOn(repo.openaiResponsesItems, 'insertMany').mockRejectedValue(new Error('simulated item persistence failure')); try { @@ -469,7 +469,7 @@ test('Responses WebSocket reports a failed turn when an output item cannot be pe } }); -test('Responses WebSocket keep-alive waits for the first event and takes a slot in the stream sequence', async () => { +test('OpenAI Responses WebSocket keep-alive waits for the first event and takes a slot in the stream sequence', async () => { const { apiKey } = await setupAppTest(); // Captured before the clock is faked: the turn's frames cross real event-loop // turns (upstream body reads, item persistence), which a faked `setTimeout` @@ -617,7 +617,7 @@ test('Responses WebSocket keep-alive waits for the first event and takes a slot ); }); -test('Responses WebSocket returns OpenAI-style error envelopes for unsupported client events', async () => { +test('OpenAI Responses WebSocket returns OpenAI-style error envelopes for unsupported client events', async () => { const { apiKey } = await setupAppTest(); await withWorkerWebSocketRuntime(async () => { const client = await connectOpenAIResponsesWebSocket(apiKey.key); @@ -638,7 +638,7 @@ test('Responses WebSocket returns OpenAI-style error envelopes for unsupported c }); }); -test('Responses WebSocket returns invalid_request_error for malformed client messages', async () => { +test('OpenAI Responses WebSocket returns invalid_request_error for malformed client messages', async () => { const { apiKey } = await setupAppTest(); await withWorkerWebSocketRuntime(async () => { const client = await connectOpenAIResponsesWebSocket(apiKey.key); @@ -719,7 +719,7 @@ test('Responses WebSocket returns invalid_request_error for malformed client mes }); }); -test('Responses WebSocket forwards HTTP failures with status, error.code, and event_id', async () => { +test('OpenAI Responses WebSocket forwards HTTP failures with status, error.code, and event_id', async () => { const { apiKey } = await setupAppTest(); await withMockedFetch( async request => { @@ -758,7 +758,7 @@ test('Responses WebSocket forwards HTTP failures with status, error.code, and ev ); }); -test('Responses WebSocket dump responseBytes counts an error envelope sent downstream', async () => { +test('OpenAI Responses WebSocket dump responseBytes counts an error envelope sent downstream', async () => { const { apiKey, repo } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const dumps = installDumpStubs(initDumpStore, initDumpBroker); @@ -802,7 +802,7 @@ test('Responses WebSocket dump responseBytes counts an error envelope sent downs ); }); -test('Responses WebSocket store:false keeps session snapshots without durable repo writes', async () => { +test('OpenAI Responses WebSocket store:false keeps session snapshots without durable repo writes', async () => { const { apiKey, repo } = await setupAppTest(); const upstreamBodies: unknown[] = []; @@ -914,7 +914,7 @@ test('Responses WebSocket store:false keeps session snapshots without durable re ); }); -test('Responses WebSocket evicts a failed continuation target so the next attempt reports previous_response_not_found', async () => { +test('OpenAI Responses WebSocket evicts a failed continuation target so the next attempt reports previous_response_not_found', async () => { const { apiKey } = await setupAppTest(); let responseCalls = 0; @@ -1018,7 +1018,7 @@ test('Responses WebSocket evicts a failed continuation target so the next attemp // client with an event instead of an error envelope, so it leaves the handler // through a different exit than the rejected turn above — and the spec's // eviction rule applies to it just the same. -test('Responses WebSocket evicts a continuation that failed through a streamed terminal event', async () => { +test('OpenAI Responses WebSocket evicts a continuation that failed through a streamed terminal event', async () => { const { apiKey } = await setupAppTest(); let responseCalls = 0; @@ -1119,7 +1119,7 @@ test('Responses WebSocket evicts a continuation that failed through a streamed t ); }); -test('Responses WebSocket store:true durable snapshots can chain through local session cache', async () => { +test('OpenAI Responses WebSocket store:true durable snapshots can chain through local session cache', async () => { const { apiKey, repo } = await setupAppTest(); let turn = 0; let firstResponseId: string | undefined; @@ -1179,7 +1179,7 @@ test('Responses WebSocket store:true durable snapshots can chain through local s assertEquals(secondSnapshot.itemIds.length > firstSnapshot.itemIds.length, true); }); -test('Responses WebSocket makes a done reasoning item reusable from a fresh connection before terminal', async () => { +test('OpenAI Responses WebSocket makes a done reasoning item reusable from a fresh connection before terminal', async () => { const { apiKey } = await setupAppTest(); const encoder = new TextEncoder(); const originalReasoning = { @@ -1287,7 +1287,7 @@ test('Responses WebSocket makes a done reasoning item reusable from a fresh conn // message resolves the prior snapshot purely from in-RAM session cache. // A fresh WS session after the repo wipe MUST NOT see it (the cache is // per-session, not per-api-key). -test('Responses WebSocket session-level store: second message resolves prior items via session cache', async () => { +test('OpenAI Responses WebSocket session-level store: second message resolves prior items via session cache', async () => { const { apiKey, repo } = await setupAppTest(); const upstreamBodies: unknown[] = []; @@ -1398,7 +1398,7 @@ test('Responses WebSocket session-level store: second message resolves prior ite ); }); -test('Responses WebSocket aborts the in-flight Responses request when the client closes', async () => { +test('OpenAI Responses WebSocket aborts the in-flight OpenAI Responses request when the client closes', async () => { const { apiKey } = await setupAppTest(); let resolveOpenAIResponsesStarted: (() => void) | undefined; const openaiResponsesStarted = new Promise(resolve => { @@ -1462,7 +1462,7 @@ test('Responses WebSocket aborts the in-flight Responses request when the client // one: a refusal whose body cannot be read. `callOpenAIResponsesUpstream` stamps this // candidate's telemetry before it dials and then reads the refusal's own words, so // the read's rejection is a mid-attempt throw with attribution already in place. -test('Responses WebSocket outer catch records a failed perf sample attributed to the throwing candidate', async () => { +test('OpenAI Responses WebSocket outer catch records a failed perf sample attributed to the throwing candidate', async () => { const { apiKey, repo, copilotUpstream } = await setupAppTest(); await withMockedFetch( @@ -1518,7 +1518,7 @@ test('Responses WebSocket outer catch records a failed perf sample attributed to assertEquals(perfRows[0]?.requests, 1); }); -test('Responses WebSocket dispatches each Codex turn with the metadata blob that turn carried', async () => { +test('OpenAI Responses WebSocket dispatches each Codex turn with the metadata blob that turn carried', async () => { const { apiKey, repo } = await setupAppTest(); await repo.upstreams.save(buildCodexUpstreamRecord()); const upstreamBodies: Record[] = []; diff --git a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-anthropic-messages_test.ts similarity index 93% rename from packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts rename to packages/gateway/__tests__/data-plane/chat/routes-anthropic-messages_test.ts index 24f9248717..2f27d3d208 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-messages_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-anthropic-messages_test.ts @@ -1,6 +1,6 @@ // `/v1/messages`, from the client's socket to Copilot's and back. // -// The same statement the Chat Completions route file makes, for the protocol Claude Code +// The same statement the OpenAI Chat Completions route file makes, for the protocol Claude Code // speaks. Two things are this family's own and are said here rather than assumed: Anthropic // ends a turn with an event rather than a transport sentinel, so `message_stop` is what the // terminal assertion looks for; and the edge writes the turn's own affinity state into the @@ -36,7 +36,7 @@ const withCopilot = async ( run, ); -/** The answer a Messages upstream gives, as the SSE every chat endpoint really speaks. */ +/** The answer an Anthropic Messages upstream gives, as the SSE every chat endpoint really speaks. */ const upstreamTurn = (model: string): Response => sseAnthropicMessagesResponse({ id: 'msg_route', @@ -70,7 +70,7 @@ interface AnthropicMessagesBody { const textOf = (body: AnthropicMessagesBody): string | undefined => body.content.find(block => block.type === 'text')?.text; -test('a streaming turn reaches the client as Messages SSE, message_stop included', async () => { +test('a streaming turn reaches the client as Anthropic Messages SSE, message_stop included', async () => { const { apiKey } = await setupAppTest(); const response = await withCopilot( @@ -106,7 +106,7 @@ test('a streaming turn reaches the client as Messages SSE, message_stop included expect(text).toBe(ANSWER); }); -test('a non-streaming turn reaches the client as one assembled Messages body', async () => { +test('a non-streaming turn reaches the client as one assembled Anthropic Messages body', async () => { const { apiKey } = await setupAppTest(); const response = await withCopilot( @@ -190,11 +190,11 @@ test('a served turn writes one usage row carrying the tokens the upstream report expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 17, output: 4 }); }); -test('a turn dialled over the Chat Completions wire still answers the client in Messages', async () => { +test('a turn dialled over the OpenAI Chat Completions wire still answers the client in Anthropic Messages', async () => { const { apiKey, repo } = await setupAppTest(); // The candidate serves `/chat/completions` and nothing else, so neither this protocol's own - // wire nor the Responses one is available and the turn leaves through the handoff. A chain + // wire nor the OpenAI Responses one is available and the turn leaves through the handoff. A chain // that dialled anything else would call a Copilot method with no upstream mocked here. const response = await withCopilot( [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts similarity index 88% rename from packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts rename to packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts index 53fba925b9..e8b9b41010 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts @@ -1,12 +1,12 @@ // `/v1beta/models/:model:generateContent`, from the client's socket to Copilot's and back. // // The same statement the other two route files make, for the one family with no wire of its -// own: the provider surface has no Gemini call, so every candidate is reached through a +// own: the provider surface has no Gemini generateContent call, so every candidate is reached through a // translation and each row here is already a translated turn. What the last row adds is the -// second pair — the same request answered over a Messages-only candidate — because "the client +// second pair — the same request answered over an Anthropic Messages-only candidate — because "the client // gets its own protocol back" has to hold for whichever wire the picker landed on. // -// Gemini also ends a turn differently from either OpenAI-shaped protocol: there is no +// Gemini generateContent also ends a turn differently from either OpenAI-shaped protocol: there is no // sentinel, and the last frame is the one whose candidate states a `finishReason`. import { expect, test } from 'vitest'; @@ -38,8 +38,8 @@ const withCopilot = async ( run, ); -/** The answer a Chat Completions upstream gives, as the SSE every chat endpoint really speaks. - * Gemini's first-preference wire, so this is what most of these turns are dialled on. */ +/** The answer an OpenAI Chat Completions upstream gives, as the SSE every chat endpoint really speaks. + * Gemini generateContent's first-preference wire, so this is what most of these turns are dialled on. */ const openaiChatCompletionsTurn = (): Response => sseOpenAIChatCompletionsResponse({ id: 'chatcmpl_route', @@ -67,15 +67,16 @@ interface GeminiGenerateContentEvent { const geminiGenerateContentText = (event: GeminiGenerateContentEvent): string => (event.candidates ?? []).flatMap(candidate => candidate.content.parts.map(part => part.text ?? '')).join(''); -test('a streaming turn reaches the client as Gemini SSE, ending on the frame that states a finish reason', async () => { +test('a streaming turn reaches the client as Gemini generateContent SSE, ending on the frame that states a finish reason', async () => { const { apiKey } = await setupAppTest(); const response = await withCopilot( [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], (path, body) => { expect(path).toBe('/chat/completions'); - // Gemini has no wire of its own, so what left the gateway is the translated turn — a - // Chat Completions body, streaming because every chat endpoint here does. + // Gemini generateContent has no wire of its own, so what left the gateway is the + // translated turn — an OpenAI Chat Completions body, streaming because every chat + // endpoint here does. expect(body.messages).toBeDefined(); expect(body.stream).toBe(true); return openaiChatCompletionsTurn(); @@ -88,7 +89,7 @@ test('a streaming turn reaches the client as Gemini SSE, ending on the frame tha const frames = parseSSEText(await response.text()); expect(frames.length > 0).toBe(true); - // One JSON object per `data:` line, and no sentinel — Gemini has none to write, so a `[DONE]` + // One JSON object per `data:` line, and no sentinel — Gemini generateContent has none to write, so a `[DONE]` // here would be a frame no Google client can parse. const events = frames.map(frame => JSON.parse(frame.data) as GeminiGenerateContentEvent); for (const frame of frames) expect(frame.event).toBe('message'); @@ -98,7 +99,7 @@ test('a streaming turn reaches the client as Gemini SSE, ending on the frame tha expect(events.slice(0, -1).every(event => event.candidates?.[0]?.finishReason === undefined)).toBe(true); }); -test('a non-streaming turn reaches the client as one assembled Gemini body', async () => { +test('a non-streaming turn reaches the client as one assembled Gemini generateContent body', async () => { const { apiKey } = await setupAppTest(); const response = await withCopilot( @@ -133,9 +134,9 @@ test('an upstream refusal reaches the client in this protocol-s own envelope', a // onto the codes it cannot express. expect(response.status).toBe(402); - // Gemini has no wire of its own, so every refusal that gets here was written by whatever + // Gemini generateContent has no wire of its own, so every refusal that gets here was written by whatever // protocol the candidate was dialled on — here OpenAI's. Handing that object on would answer - // a Gemini client in another protocol's words: `error.status` is the field its SDKs read and + // a Gemini generateContent client in another protocol's words: `error.status` is the field its SDKs read and // an OpenAI envelope has none, while `error.code` would arrive as a string where a number // belongs. So the upstream's words become the message and this protocol writes the shape. const body = await response.json() as { error: { code: number; message: string; status: string } }; @@ -151,7 +152,7 @@ test('a refusal on a turn that asked to stream is a body, not an opened stream', const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota' } }; // The riskier half of the same statement: the client asked for SSE, so the seam has a - // stream to open and must not — nothing was ever generated, and a Gemini stream that + // stream to open and must not — nothing was ever generated, and a Gemini generateContent stream that // carried the refusal would be read as a turn that produced no candidates. const response = await withCopilot( [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], @@ -194,7 +195,7 @@ test('a served turn writes one usage row carrying the tokens the upstream report expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 21, output: 3 }); }); -test('a turn dialled over the Messages wire still answers the client in Gemini', async () => { +test('a turn dialled over the Anthropic Messages wire still answers the client in Gemini generateContent', async () => { const { apiKey, repo } = await setupAppTest(); // The candidate serves `/messages` and nothing else, so the picker's first preference is diff --git a/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts similarity index 95% rename from packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts rename to packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts index 6c794053e8..6517ea0a8c 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-chat-completions_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts @@ -36,7 +36,7 @@ const withCopilot = async ( run, ); -/** The answer a Chat Completions upstream gives, as the SSE every chat endpoint really speaks. */ +/** The answer an OpenAI Chat Completions upstream gives, as the SSE every chat endpoint really speaks. */ const upstreamTurn = (model: string): Response => sseOpenAIChatCompletionsResponse({ id: 'chatcmpl_route', @@ -59,7 +59,7 @@ interface ChatChunk { readonly choices: readonly { readonly delta?: { readonly content?: string }; readonly finish_reason: string | null }[]; } -test('a streaming turn reaches the client as Chat Completions SSE, terminator included', async () => { +test('a streaming turn reaches the client as OpenAI Chat Completions SSE, terminator included', async () => { const { apiKey } = await setupAppTest(); const response = await withCopilot( @@ -90,7 +90,7 @@ test('a streaming turn reaches the client as Chat Completions SSE, terminator in expect(chunks.some(chunk => chunk.choices[0]?.finish_reason === 'stop')).toBe(true); }); -test('a non-streaming turn reaches the client as one assembled Chat Completions body', async () => { +test('a non-streaming turn reaches the client as one assembled OpenAI Chat Completions body', async () => { const { apiKey } = await setupAppTest(); const response = await withCopilot( @@ -179,7 +179,7 @@ test('a served turn writes one usage row carrying the tokens the upstream report expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 11, output: 7 }); }); -test('a turn dialled over the Messages wire still answers the client in Chat Completions', async () => { +test('a turn dialled over the Anthropic Messages wire still answers the client in OpenAI Chat Completions', async () => { const { apiKey, repo } = await setupAppTest(); // The candidate serves `/messages` and nothing else, so the picker's first preference — this diff --git a/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts similarity index 97% rename from packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts rename to packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts index 172aaecd80..2c42260609 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses-routes_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts @@ -43,7 +43,7 @@ const post = async (apiKey: string, body: Record): Promise { +test('a turn reaches Copilot over its own OpenAI Responses endpoint and comes back as one response', async () => { const { apiKey } = await setupAppTest(); let sent: Record | undefined; diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 00018416b0..36125ef410 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -255,7 +255,7 @@ beforeEach(() => { }); describe('a chat family reaching a candidate over another protocol', () => { - it('serves /v1/chat/completions on a Messages-only candidate', async () => { + it('serves /v1/chat/completions on an Anthropic Messages-only candidate', async () => { const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); @@ -267,7 +267,7 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); - it('serves /v1/chat/completions on a Responses-only candidate', async () => { + it('serves /v1/chat/completions on an OpenAI Responses-only candidate', async () => { const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); @@ -278,7 +278,7 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); - it('serves /v1/messages on a Responses-only candidate', async () => { + it('serves /v1/messages on an OpenAI Responses-only candidate', async () => { const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); @@ -289,7 +289,7 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); - it('serves /v1/messages on a Chat Completions-only candidate', async () => { + it('serves /v1/messages on an OpenAI Chat Completions-only candidate', async () => { const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); @@ -300,7 +300,7 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); - it('serves /v1/responses on a Messages-only candidate', async () => { + it('serves /v1/responses on an Anthropic Messages-only candidate', async () => { const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); @@ -311,7 +311,7 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); - it('serves /v1/responses on a Chat Completions-only candidate', async () => { + it('serves /v1/responses on an OpenAI Chat Completions-only candidate', async () => { const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); @@ -322,8 +322,8 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); - // Gemini has no wire of its own, so all three of its rows are translated ones. - it('serves :generateContent on a Chat Completions-only candidate', async () => { + // Gemini generateContent has no wire of its own, so all three of its rows are translated ones. + it('serves :generateContent on an OpenAI Chat Completions-only candidate', async () => { const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); @@ -334,7 +334,7 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); - it('serves :generateContent on a Messages-only candidate', async () => { + it('serves :generateContent on an Anthropic Messages-only candidate', async () => { const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); @@ -345,7 +345,7 @@ describe('a chat family reaching a candidate over another protocol', () => { await drain(); }); - it('serves :generateContent on a Responses-only candidate', async () => { + it('serves :generateContent on an OpenAI Responses-only candidate', async () => { const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); @@ -436,11 +436,11 @@ describe('the fork over wires', () => { }); describe('a rule that speaks about one protocol-s wire', () => { - // The Messages fold rewrites *every* inline system message, because Anthropic's top-level - // `system` is the only first-position slot there. The Chat Completions fold rewrites only a + // The Anthropic Messages fold rewrites *every* inline system message, because Anthropic's top-level + // `system` is the only first-position slot there. The OpenAI Chat Completions fold rewrites only a // system message past the leading run. So a leading system message tells the two apart: a - // turn that left for Chat Completions must be folded by the Chat Completions rule, which - // leaves it alone, and not by the Messages one, which would not. + // turn that left for OpenAI Chat Completions must be folded by the OpenAI Chat Completions rule, which + // leaves it alone, and not by the Anthropic Messages one, which would not. it('does not run on a turn that leaves for another protocol', async () => { let sent: { messages: { role: string }[] } | undefined; const payload = { @@ -467,8 +467,8 @@ describe('a rule that speaks about one protocol-s wire', () => { }); // The other direction, and the one that pays for the whole arrangement: the usage chunk is - // asked for by the Chat Completions wire, so a Messages turn that reaches an upstream over - // that endpoint is metered like any other turn on it. A rule left in the Chat Completions + // asked for by the OpenAI Chat Completions wire, so an Anthropic Messages turn that reaches an upstream over + // that endpoint is metered like any other turn on it. A rule left in the OpenAI Chat Completions // *source* chain would ask for nothing here, and this turn would bill zero. it('asks for the usage chunk on a turn that arrived from another protocol', async () => { let sent: { stream_options?: unknown } | undefined; @@ -487,7 +487,7 @@ describe('a rule that speaks about one protocol-s wire', () => { // The same arrangement for a rule no guard ever covered. The old surface ran the *whole* // target array on a translated body, so a turn arriving over a translation was shaped for - // the wire it landed on. A rule left in the Messages source chain would scrub nothing here. + // the wire it landed on. A rule left in the Anthropic Messages source chain would scrub nothing here. it('applies the target wire-s unguarded rules to a turn that arrived translated', async () => { let sent: { system?: unknown } | undefined; resolves([candidate('up_a', { anthropicMessages: {} }, { @@ -509,8 +509,9 @@ describe('a rule that speaks about one protocol-s wire', () => { await drain(); }); - // And a rule the *other* wire owns stays off this turn. `stream_options` is a Chat - // Completions field, so a Chat Completions turn dialled over Messages must not carry it — + // And a rule the *other* wire owns stays off this turn. `stream_options` is an OpenAI Chat + // Completions field, so an OpenAI Chat Completions turn dialled over Anthropic Messages must + // not carry it — // which is what the interceptor form said with `ctx.targetApi !== 'openaiChatCompletions'` and // what position says here. it('does not carry another wire-s request rules onto the wire it dialled', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts index 0639611f12..76c885f710 100644 --- a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts @@ -10,9 +10,9 @@ // express: the verdict belongs to one candidate. The next candidate may be reachable on a wire // whose protocol *can* carry the body, and failover re-runs the suffix to find out. // -// Both of Gemini's actions cross a protocol boundary and both are covered: generation reaches +// Both of Gemini generateContent's actions cross a protocol boundary and both are covered: generation reaches // its target through a handoff, and `:countTokens` through the pair that asks the question in -// Messages. The replaced surface answered a refusal the same way on either, because what the +// Anthropic Messages. The replaced surface answered a refusal the same way on either, because what the // caller reads is the same protocol whichever action it asked for. import { test } from 'vitest'; @@ -30,7 +30,7 @@ const upstream = (models: Parameters[0]) => async (request throw new Error(`Unhandled fetch ${request.url}`); }; -// A `functionResponse` part in model content: a shape Gemini defines and neither target protocol +// A `functionResponse` part in model content: a shape Gemini generateContent defines and neither target protocol // has anywhere to put, so the translator refuses it rather than dropping it. const untranslatableGeminiGenerateContentBody = { contents: [ @@ -39,7 +39,7 @@ const untranslatableGeminiGenerateContentBody = { ], }; -/** What a Gemini client reads a refusal out of. `error.status` is the field it keys on, and it +/** What a Gemini generateContent client reads a refusal out of. `error.status` is the field it keys on, and it * is not a field the gateway's internal-error envelope has — nor is `stack` a field a * caller-input refusal has any business carrying. */ const assertGoogleRpcRefusal = async (response: Response): Promise => { @@ -73,7 +73,7 @@ test('a body the counting wire cannot carry is refused in the client own protoco const { apiKey } = await setupAppTest(); await withMockedFetch( - // Counting is reachable only over an upstream's own Messages endpoint, so this is the one + // Counting is reachable only over an upstream's own Anthropic Messages endpoint, so this is the one // candidate shape that gets as far as the translation. upstream([{ id: 'claude-count', supported_endpoints: ['/messages'] }]), async () => { diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 87b4a21a91..d20ad8ad02 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -33,7 +33,7 @@ const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] ['Gemini generateContent', () => geminiGenerateContentServePipeline({ model: 'm', contents: [] } as never)], ['OpenAI Responses', () => openaiResponsesServePipeline({ model: 'm', input: [] } as never)], ['Anthropic Messages count_tokens', () => anthropicMessagesCountTokensPipeline({ model: 'm', messages: [] } as never)], - ['Gemini countTokens', () => geminiGenerateContentCountTokensPipeline({ model: 'm', contents: [] } as never)], + ['Gemini generateContent countTokens', () => geminiGenerateContentCountTokensPipeline({ model: 'm', contents: [] } as never)], ['OpenAI Responses compact', () => openaiResponsesCompactPipeline({ model: 'm', input: [] } as never)], ]; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index 0221606b21..3c7ad0dda2 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -1,10 +1,10 @@ // `/v1/messages/count_tokens` as a pipeline. // -// emitAnthropicMessagesTokenCount the edge: writes the measurement, or the refusal, as one body -// writeSettlement above the fork, so a measurement is sampled once however many it tried -// resolveChatCandidates narrows to what can serve, in the order affinity asks for -// failover runs what follows once per candidate -// materializeAttempt puts the payload this candidate is owed into the record +// emitAnthropicMessagesTokenCount the edge: writes the measurement, or the refusal, as one body +// writeSettlement above the fork, so a measurement is sampled once however many it tried +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// materializeAttempt puts the payload this candidate is owed into the record // the three request rules generation applies, and the web-search request preparation // callAnthropicMessagesCountTokensUpstream the ending: measures, and never opens a stream // @@ -15,10 +15,10 @@ // owes and no usage row. Measuring is not generating, and an upstream that answered the // question charged nothing for it. // -// There is one wire, and it is shared. Only an upstream's own Messages endpoint measures — +// There is one wire, and it is shared. Only an upstream's own Anthropic Messages endpoint measures — // no protocol but this one answers "what would this cost" — so the chain dials directly // instead of forking, and Gemini's `:countTokens` translates into the same wire rather than -// growing one of its own. The rule generation keeps inside its Messages wire (the system-role +// growing one of its own. The rule generation keeps inside its Anthropic Messages wire (the system-role // rewrite, which speaks about what that endpoint accepts) is therefore a rule of this wire // too, and sits with it. // @@ -50,7 +50,7 @@ import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-d import { parseAnthropicBetaHeader, type AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import { providerModelOf } from '@floway-dev/provider'; -/** Counting has no translation path: only an upstream's own Messages endpoint can answer the +/** Counting has no translation path: only an upstream's own Anthropic Messages endpoint can answer the * question, so a candidate that would serve generation over a translated wire cannot serve * this. */ export const anthropicMessagesCountTokensTarget = chatTargetPicker(['anthropicMessages']); @@ -246,13 +246,13 @@ const callAnthropicMessagesCountTokensUpstream = defineStage< }); /** - * The Messages count-tokens wire, as the chain that dials it. + * The Anthropic Messages count-tokens wire, as the chain that dials it. * * Every source protocol whose measurement reaches an upstream does it over this endpoint — * Gemini's `:countTokens` has no wire of its own and arrives here through a translation — so * a rule that speaks about *this* wire belongs here rather than beside a source's own * strippers. All four do: the system-role rewrite and the reasoning sentinel state what an - * upstream's Messages endpoint accepts, the billing-attribution scrub states what generation + * upstream's Anthropic Messages endpoint accepts, the billing-attribution scrub states what generation * would have sent it, and the web-search preparation states the tool shape it would have * seen. */ diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts index 2ea9c99459..a3f2400c2d 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts @@ -4,7 +4,7 @@ import { generateAnthropicId } from '@floway-dev/protocols/anthropic-messages'; // `{type:'error', error:{type, message}, request_id}` with `request_id` at the // top level — alongside `error`, not nested inside it — and the key order is // load-bearing for byte-faithfulness. -/** What a Messages client is sent when a turn produced no content. The envelope is +/** What an Anthropic Messages client is sent when a turn produced no content. The envelope is * Anthropic's own — `type` at the top, the error's own `type` inside, and a `request_id` a * caller quotes back in a support thread — so it is the protocol's shape rather than the * gateway's, and an OpenAI-shaped one would carry none of those fields. */ diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts index 505d2c62c5..f32422ce3e 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts @@ -17,7 +17,7 @@ import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/s import { move } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -// Reject `anthropic_beta` / `betas` in the body; the Messages protocol carries +// Reject `anthropic_beta` / `betas` in the body; the Anthropic Messages protocol carries // them via the `anthropic-beta` HTTP header. const rejectBodyBetaResponse = (payload: AnthropicMessagesPayload): Response | null => { const record = payload as unknown as Record; @@ -30,7 +30,7 @@ const rejectBodyBetaResponse = (payload: AnthropicMessagesPayload): Response | n return Response.json( { error: { - message: `${param} in the Messages request body is not supported; send Anthropic beta flags with the anthropic-beta HTTP header.`, + message: `${param} in the Anthropic Messages request body is not supported; send Anthropic beta flags with the anthropic-beta HTTP header.`, type: 'invalid_request_error', param, }, diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts index 7a8a61fff7..73932e59b8 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts @@ -24,7 +24,7 @@ import { eventResult, providerModelOf, type ExecuteResult, type ModelCandidate } // // A one-token cap is not portable. OpenAI's Responses API floors // `max_output_tokens` at 16 and rejects anything lower with a hard 400, so -// every Messages→Responses candidate fails the probe and Claude Code +// every Anthropic Messages→OpenAI Responses candidate fails the probe and Claude Code // concludes the model does not exist — the CLI surfaces the upstream envelope // verbatim: `API error: 400 {"error":{"message":"Invalid 'max_output_tokens': // integer below minimum value. Expected a value >= 16, but got 1 instead.", diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts index 32bc200344..0facbf88e6 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts @@ -372,7 +372,7 @@ const prepareAnthropicMessagesWebSearchReplay = (messages: AnthropicMessagesMess // System messages with array content pass through unchanged: the // remaining rewrite path below assumes assistant-shape content // (server_tool_use, web_search_tool_result, citations) and finalizes - // with `role: 'assistant'`, which would silently corrupt a + // with `role: 'assistant'`, which would silently corrupt an // AnthropicMessagesSystemMessage carrying AnthropicMessagesTextBlock[] into an assistant // turn. System messages never own web-search replay markers. if (message.role === 'system') { diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 4fdfa430c8..ab1ec9c5e2 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -1,6 +1,6 @@ -// Messages as a pipeline, on the chain Chat Completions established. +// Anthropic Messages as a pipeline, on the chain OpenAI Chat Completions established. // -// emitAnthropicMessages the edge: writes the answer in the shape the client asked for +// emitAnthropicMessages the edge: writes the answer in the shape the client asked for // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate @@ -9,7 +9,7 @@ // dialChatWire the ending: picks this candidate's wire and hands into it // // Three wires, all handing up `response.chat.anthropicMessages`: this protocol's own, and the two -// translated ones — Messages via Responses and Messages via Chat Completions — each a +// translated ones — Anthropic Messages via OpenAI Responses and via OpenAI Chat Completions — each a // handoff followed by that protocol's own wire. The stage above cannot tell which ran. // // `/v1/messages/count_tokens` is not one of them: it is a second operation over this @@ -63,8 +63,8 @@ import { import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateAnthropicMessagesViaOpenAIChatCompletions, translateAnthropicMessagesViaOpenAIResponses } from '@floway-dev/translate'; -/** `/v1/messages` prefers its own wire, then the translated Responses path, then the - * translated Chat Completions path. */ +/** `/v1/messages` prefers its own wire, then the translated OpenAI Responses path, then the + * translated OpenAI Chat Completions path. */ export const anthropicMessagesTarget = chatTargetPicker(['anthropicMessages', 'openaiResponses', 'openaiChatCompletions']); /** What this family adds to the chat space. */ @@ -80,7 +80,7 @@ export interface AnthropicMessagesFacts extends ChatFacts { type M = { [P in K]: AnthropicMessagesFacts[P] }; /** - * The outermost edge. A Messages answer is always a stream by the time it reaches here — + * The outermost edge. An Anthropic Messages answer is always a stream by the time it reaches here — * the upstream speaks SSE whatever the client asked for — so what this decides is whether * the client sees the frames or the one message they add up to. * @@ -163,7 +163,7 @@ const emitAnthropicMessages = defineStage< /** Anthropic names its own SSE events, and every frame that has one is the client's. Which * frames have an SSE form at all is the protocol's to say, and it says so by writing no - * frame — a stream terminator is a Chat Completions idea and there is nothing to send for + * frame — a stream terminator is an OpenAI Chat Completions idea and there is nothing to send for * it here. */ const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ [Symbol.asyncIterator]: () => (async function* () { @@ -175,7 +175,7 @@ const renderSSE = (frames: AsyncIterable defineStage< }); /** - * The Messages wire, as the chain that dials it. + * The Anthropic Messages wire, as the chain that dials it. * * Every source protocol that reaches an upstream over this endpoint runs this, whether the - * client spoke Messages or a handoff arrived here — which is what makes a rule that speaks - * about *this* wire belong here. The system-role rewrite states what an upstream's Messages + * client spoke Anthropic Messages or a handoff arrived here — which is what makes a rule that + * speaks about *this* wire belong here. The system-role rewrite states what an upstream's Anthropic Messages * endpoint accepts, so it applies to whatever body this wire actually sends and to nothing * that leaves for another protocol. */ @@ -442,7 +442,7 @@ const meterAnthropicMessages = ( return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; }; -/** What ends a Messages turn. Anthropic's own stream terminator is an event rather than a +/** What ends an Anthropic Messages turn. Anthropic's own stream terminator is an event rather than a * transport sentinel, and a stream that failed mid-turn says so with `error` in place of the * `message_stop` that will now never come. */ const isAnthropicMessagesTerminalFrame = (frame: ProtocolFrame): boolean => diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts index f4d18cab64..23ec8bf417 100644 --- a/packages/gateway/src/data-plane/chat/facts.ts +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -16,7 +16,7 @@ import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini- import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -/** The four protocols a client can speak here. Gemini is source-only: nothing translates +/** The four protocols a client can speak here. Gemini generateContent is source-only: nothing translates * into it, so by the ruling it contributes one pipeline rather than two — a source role * and no target role. */ export type ChatSourceProtocol = 'openaiChatCompletions' | 'anthropicMessages' | 'openaiResponses' | 'geminiGenerateContent'; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index fe948448e1..a4cc3bfec9 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -1,4 +1,4 @@ -// Gemini's `:countTokens` as a pipeline. +// Gemini generateContent's `:countTokens` as a pipeline. // // emitGeminiGenerateContentTokenCount the edge: writes the measurement, or the refusal, as one body // writeSettlement above the fork, so a measurement is sampled once however many it tried @@ -6,13 +6,15 @@ // failover runs what follows once per candidate // materializeAttempt puts the payload this candidate is owed into the record // the three strippers the same protocol-shape cleanups generation applies -// measureGeminiGenerateContentAsAnthropicMessages the pair: asks the question in Messages and reads the answer back -// the Messages count wire the ending, shared with `/v1/messages/count_tokens` +// measureGeminiGenerateContentAsAnthropicMessages the pair: asks the question in Anthropic +// Messages and reads the answer back +// the Anthropic Messages count wire the ending, shared with +// `/v1/messages/count_tokens` // // A second **operation** over this protocol rather than another wire under // `geminiGenerateContentServePipeline`, so it is a chain of its own. Like generation it has no wire of its -// own — the provider surface has no Gemini call — but unlike generation it has only one -// target: no protocol but Messages answers "what would this cost", so there is nothing to +// own — the provider surface has no Gemini generateContent call — but unlike generation it has only one +// target: no protocol but Anthropic Messages answers "what would this cost", so there is nothing to // pick between and the pair is in the array rather than under a fork. // // The pair is not `handOff`. A handoff maps frames, and there are none here: what crosses the @@ -45,7 +47,7 @@ import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline' import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; import { translateGeminiGenerateContentViaAnthropicMessages, TranslatorInputError } from '@floway-dev/translate'; -/** Counting is reachable only over an upstream's own Messages endpoint: no other protocol +/** Counting is reachable only over an upstream's own Anthropic Messages endpoint: no other protocol * answers the question, and no translation invents an answer. */ export const geminiGenerateContentCountTokensTarget = chatTargetPicker(['anthropicMessages']); @@ -101,10 +103,10 @@ const emitGeminiGenerateContentTokenCount = defineStage< }); /** - * Asks the question in Messages, and reads the answer back out as Google's. + * Asks the question in Anthropic Messages, and reads the answer back out as Google's. * * It consumes this protocol's request key and provides the target's, exactly as a handoff - * does, so the wire below sees only Messages and cannot tell it was reached by translation. + * does, so the wire below sees only Anthropic Messages and cannot tell it was reached by translation. * What differs is the way back: there are no frames to map, only counts. * * It carries the `return` trait for the same reason a handoff does — a body the target diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts index 756baadc10..b807729d8d 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts @@ -1,6 +1,6 @@ // Google RPC Status envelope, used by Gemini's `error` channel everywhere // (HTTP body, SSE-tunnelled error event). -/** What a Gemini client is sent when a turn produced no content. `error.status` is the +/** What a Gemini generateContent client is sent when a turn produced no content. `error.status` is the * Google-RPC name and clients read it, so this shape is the protocol's rather than the * gateway's — an OpenAI-shaped envelope would leave that field undefined. */ export const renderGeminiGenerateContentError = (status: number, message: string): Record => ({ diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts index bc81900573..d0612a9147 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts @@ -22,7 +22,7 @@ interface GeminiGenerateContentModelAction { readonly action: string; } -// The Gemini wire API encodes both the model id and the action in one path +// The Gemini generateContent wire API encodes both the model id and the action in one path // segment (e.g. `models/gemini-2.5-pro:streamGenerateContent`). The Hono route // captures everything after `/v1beta/models/` in a single `modelAction` param; // we split on the trailing `:` here so each entry sees just the action and @@ -101,7 +101,7 @@ const runGeminiGenerateContentGenerate = async (c: AuthedContext, model: string, 'ingress.chat.sourceProtocol': 'geminiGenerateContent', 'ingress.chat.geminiGenerateContent.wantsStream': wantsStream, 'request.chat.geminiGenerateContent': payload, - // Gemini carries the model in the path rather than the body, so the id the run + // Gemini generateContent carries the model in the path rather than the body, so the id the run // resolves against is the one the route split off. 'serve.model': model, }) as never, diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index f1f19fd9fe..d42e547bec 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -1,28 +1,29 @@ -// Gemini as a pipeline — the same chain as Chat Completions, for the one source-only family. +// Gemini generateContent as a pipeline — the same chain as OpenAI Chat Completions, for the +// one source-only family. // -// emitGeminiGenerateContent the edge: writes the answer in the shape the client asked for -// writeSettlement above the fork, so a run bills once however many wires it tried -// resolveChatCandidates narrows to what can serve, in the order affinity asks for -// failover runs what follows once per candidate -// materializeAttempt puts the payload this candidate is owed into the record +// emitGeminiGenerateContent the edge: writes the answer in the shape the client asked for +// writeSettlement above the fork, so a run bills once however many wires it tried +// resolveChatCandidates narrows to what can serve, in the order affinity asks for +// failover runs what follows once per candidate +// materializeAttempt puts the payload this candidate is owed into the record // the strippers, then requireGeminiGenerateContentTerminal -// dialChatWire the ending: picks this candidate's wire and hands into it +// dialChatWire the ending: picks this candidate's wire and hands into it // -// Gemini is source-only. Nothing translates *into* it, so it contributes one pipeline rather +// Gemini generateContent is source-only. Nothing translates *into* it, so it contributes one pipeline rather // than two — a source role and no target role, and no other family's chain ever hands up // `response.chat.geminiGenerateContent`. // // It is also the one family with no wire of its own: the provider surface has no `callGeminiGenerateContent`, // so every candidate is reached through a translation and all three wires are translated ones -// — a handoff into Chat Completions, into Messages or into Responses, followed by that +// — a handoff into OpenAI Chat Completions, into Anthropic Messages or into OpenAI Responses, followed by that // protocol's own wire. Which one a candidate is dialled on is the picker's answer for that // candidate, and failover re-running the suffix is what re-asks for the next. // // Because every wire is translated, the reading is taken on the dialect the upstream actually // spoke: the target protocol's own wire meters its own frames on their way into the // translation, so a figure the upstream stated is billed as stated. Where the *turn* ends is -// a different question and is stated on the Gemini frames the client is handed, which is what -// `requireGeminiGenerateContentTerminal` is for — a Chat Completions stream that closed cleanly without ever +// a different question and is stated on the Gemini generateContent frames the client is handed, +// which is what `requireGeminiGenerateContentTerminal` is for — an OpenAI Chat Completions stream that closed cleanly without ever // reporting a finish reason translates into candidates that never finish. // // `:countTokens` is not one of the three: it is a second operation over this protocol rather @@ -30,7 +31,7 @@ // `{ totalTokens }` envelope of its own — so it is a chain of its own in `count-tokens.ts`. // // What is not built, stated rather than implied: -// - the affinity egress' thought-signature rewrite is here, but the Gemini interceptors are +// - the affinity egress' thought-signature rewrite is here, but the Gemini generateContent interceptors are // only partly stages: the four `interceptors/` entries are, and nothing else is. // - the Google-RPC envelope `respond.ts` wrote into a client's own stream for a turn that // ended without a terminal event. The chain detects one and leaves the run as a throw @@ -38,8 +39,8 @@ // // A refusal is the one place where having no wire of its own changes what a client is owed. // Every other family can hand an upstream's own error object on, because the upstream spoke -// that family's protocol. Here it never did: whatever came back was written by Chat -// Completions, Messages or Responses, and `error.status` — the field a Google client reads — +// that family's protocol. Here it never did: whatever came back was written by OpenAI Chat +// Completions, Anthropic Messages or OpenAI Responses, and `error.status` — the field a Google client reads — // is in none of them. So the words survive and the shape is this protocol's, which is what // the handoff arranges by dropping a foreign envelope on the way up. // @@ -84,7 +85,7 @@ import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; import { translateGeminiGenerateContentViaOpenAIChatCompletions, translateGeminiGenerateContentViaAnthropicMessages, translateGeminiGenerateContentViaOpenAIResponses } from '@floway-dev/translate'; /** `:generateContent` has no wire of its own, so the whole preference list is translated: - * Chat Completions first, then Messages, then Responses. */ + * OpenAI Chat Completions first, then Anthropic Messages, then OpenAI Responses. */ export const geminiGenerateContentTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); /** What this family adds to the chat space. */ @@ -100,8 +101,8 @@ export interface GeminiGenerateContentFacts extends ChatFacts { type C = { [P in K]: GeminiGenerateContentFacts[P] }; /** - * The outermost edge. A Gemini answer is always a stream by the time it reaches here — the - * wire below speaks SSE whatever the client asked for, and the translation back into Gemini + * The outermost edge. A Gemini generateContent answer is always a stream by the time it reaches + * here — the wire below speaks SSE whatever the client asked for, and the translation back into Gemini generateContent * is itself a stream of frames — so what this decides is whether the client sees the frames * or the one object they add up to. * @@ -184,7 +185,7 @@ const emitGeminiGenerateContent = defineStage< }, }); -/** Gemini streams one JSON object per `data:` line and has no sentinel to write, which the +/** Gemini generateContent streams one JSON object per `data:` line and has no sentinel to write, which the * protocol says by writing no frame at all for the one that ends the stream. */ const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ [Symbol.asyncIterator]: () => (async function* () { @@ -196,10 +197,10 @@ const renderSSE = (frames: AsyncIterable { const context = { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens }; switch (target) { diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index 64d56c3bff..095f83cc20 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -146,7 +146,7 @@ export const handOff = { let crossedLeadingSystemRun = false; @@ -326,7 +326,7 @@ export const stripPromptCacheKeyForOpenAIChatCompletions = defineStage< /** * Asks the upstream for the usage chunk that billing is read off. * - * Chat Completions emits the final usage-only chunk only when `stream_options.include_usage` + * OpenAI Chat Completions emits the final usage-only chunk only when `stream_options.include_usage` * is on, and the gateway meters every stream from those frames — so what the client asked for * and what the upstream is asked for differ here, which is the one thing this rule exists to * say. Nothing downstream has to thread the client's original value through, because it is a @@ -335,7 +335,7 @@ export const stripPromptCacheKeyForOpenAIChatCompletions = defineStage< * * It belongs to the wire rather than to the source chain because it names a field of *this* * protocol's request. Every source protocol that reaches an upstream over this endpoint runs - * it — a Messages or Gemini turn served here would otherwise be metered off a stream that was + * it — an Anthropic Messages or Gemini generateContent turn served here would otherwise be metered off a stream that was * never asked to report anything — and a turn that leaves for another wire gets that wire's * own rule for the same thing. * @@ -448,7 +448,7 @@ const withUsageOnItsOwnCarrier = async function* ( * `total_tokens` as the witness, and the `usage-exclusive-cached-tokens` flag is a declaration * input for the responses whose totals witness nothing. * - * Everything it says is about one upstream's Chat Completions wire — the flag it reads + * Everything it says is about one upstream's OpenAI Chat Completions wire — the flag it reads * describes how that upstream writes its usage there, and the remedy its errors name is a * setting for that upstream. That is why it belongs to the wire: on any other wire these * events are a projection of some other protocol, where the flag answers a question about @@ -501,7 +501,7 @@ export const normalizeExclusiveCachedTokensForOpenAIChatCompletions = defineStag }), }); -/** Chat Completions carries usage on the chunk's own root and names the buckets +/** OpenAI Chat Completions carries usage on the chunk's own root and names the buckets * `prompt_tokens` and `prompt_tokens_details.{cached_tokens, cache_creation_input_tokens}`. */ const OPENAI_CHAT_COMPLETIONS_CACHE_BUCKETS: CacheBucketNames = { input: 'prompt_tokens', @@ -521,7 +521,7 @@ const foldOpenAIChatCompletionsUsage = ( return folded === usage ? chunk : { ...chunk, usage: folded as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; }; -// ── Chat Completions vendor dialects ────────────────────────────────────────────────────── +// ── OpenAI Chat Completions vendor dialects ─────────────────────────────────────────────── // // Each of the three below is last among this wire's rewrites, which is what gives it the final // say on the outbound body and the first say on the inbound stream: everything above deals @@ -753,7 +753,7 @@ const kimiInboundUsage = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCo return { ...chunk, usage: next as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; }; -/** None of the three fields is a Chat Completions field, so each is declared beside the vendor +/** None of the three fields is an OpenAI Chat Completions field, so each is declared beside the vendor * that reads it rather than widening the protocol's own types. */ type OpenAIChatCompletionsPayloadWithDeepSeekThinking = Omit & { thinking: { type: string } }; type OpenAIChatCompletionsPayloadWithQwenThinking = Omit & { enable_thinking: false }; @@ -762,7 +762,7 @@ type OpenAIChatCompletionsMessageWithDeepSeekReasoning = type OpenAIChatCompletionsDeltaWithDeepSeekReasoning = OpenAIChatCompletionsStreamEvent['choices'][number]['delta'] & { reasoning_content?: unknown }; -// ── Messages ────────────────────────────────────────────────────────────────────────────── +// ── Anthropic Messages ──────────────────────────────────────────────────────────────────── /** * The mid-conversation system rewrite, which on this protocol is every inline system message. @@ -940,15 +940,15 @@ const scrubbedSystemPrompt = (system: AnthropicMessagesPayload['system']): Anthr return blocks.length > 0 ? blocks : undefined; }; -// ── Gemini ──────────────────────────────────────────────────────────────────────────────── +// ── Gemini generateContent ──────────────────────────────────────────────────────────────── // // The three strippers below are unconditional rather than flag-gated, and that is a statement -// about the target graph rather than about any upstream: Gemini has no wire of its own here, +// about the target graph rather than about any upstream: Gemini generateContent has no wire of its own here, // so every turn is served through a translation, and what no translation can carry cannot be // sent whichever candidate answers. What was `delete ctx.payload.x` is a rewrite now — the // record is frozen, so a stage that deleted in place would throw rather than strip. -/** Gemini file and code parts have no equivalent anywhere the translations reach, so they go +/** Gemini generateContent file and code parts have no equivalent anywhere the translations reach, so they go * at source and every target sees translatable parts. A part left holding nothing goes with * them: it is not a part any more. */ export const stripUnsupportedPartFieldsFromGeminiGenerateContent = defineStage< @@ -1010,7 +1010,7 @@ const stripContentParts = (content: GeminiGenerateContentContent): GeminiGenerat return changed ? { ...content, parts } : content; }; -/** Only function declarations translate out of a Gemini tool group, so the rest of a group's +/** Only function declarations translate out of a Gemini generateContent tool group, so the rest of a group's * capabilities go — and a group left declaring no function goes with them, because a target * emitter offered an empty group would be offered a tool that does nothing. `tools` itself * goes when no group survived: an empty tool list is a different request from no tools. */ @@ -1039,7 +1039,7 @@ export const stripUnsupportedToolsFromGeminiGenerateContent = defineStage< })), }); -/** Every field a Gemini tool group can carry other than `functionDeclarations` — the +/** Every field a Gemini generateContent tool group can carry other than `functionDeclarations` — the * capabilities Google executes inside its own generation loop, which no target wire can be * asked to run. `functionDeclarations` is the one that survives because the model "does not * execute the function"; it asks the caller to. Re-derive against the `Tool` schema whenever @@ -1072,7 +1072,7 @@ const withUnsupportedToolsStripped = (payload: GeminiGenerateContentPayload): Ge return changed ? { ...payload, tools: kept } : payload; }; -/** Gemini safety controls are source-specific and have no matching control on every target +/** Gemini generateContent safety controls are source-specific and have no matching control on every target * path, so they go rather than have us pretend to enforce a policy we cannot honor * end to end. */ export const stripSafetySettingsFromGeminiGenerateContent = defineStage< @@ -1101,7 +1101,7 @@ export const stripSafetySettingsFromGeminiGenerateContent = defineStage< }); /** - * Hides Gemini thought-summary parts from a caller who did not ask for them. + * Hides Gemini generateContent thought-summary parts from a caller who did not ask for them. * * The opt-in is `generationConfig.thinkingConfig.includeThoughts`, which is something the * client sent — a request-direction reading — and the rule that uses it runs on the way back. @@ -1187,9 +1187,9 @@ const hasGeminiGenerateContentEventPayload = (event: GeminiGenerateContentStream || event.responseId !== undefined; }; -// ── Responses ───────────────────────────────────────────────────────────────────────────── +// ── OpenAI Responses ────────────────────────────────────────────────────────────────────── -/** Role rewrites, in the same fixed order and by the same fold as Chat Completions; what is +/** Role rewrites, in the same fixed order and by the same fold as OpenAI Chat Completions; what is * here is the walk over this protocol's input items. Only a message item carries a role, and * an item that carries none still crosses the leading system run — a reasoning item between * two system messages is what makes the second one mid-conversation. */ @@ -1361,7 +1361,7 @@ export const normalizeExclusiveCachedTokensForOpenAIResponses = defineStage< }), }); -/** Responses carries usage on `event.response.usage` and names the buckets `input_tokens` and +/** OpenAI Responses carries usage on `event.response.usage` and names the buckets `input_tokens` and * `input_tokens_details.{cached_tokens, cache_write_tokens}`. */ const OPENAI_RESPONSES_CACHE_BUCKETS: CacheBucketNames = { input: 'input_tokens', @@ -1454,7 +1454,7 @@ export const vendorQwenNormalizeForOpenAIResponses = defineStage< })), }); -/** Neither field is a Responses field, so each is declared beside the vendor that reads it +/** Neither field is an OpenAI Responses field, so each is declared beside the vendor that reads it * rather than widening the protocol's own request type. */ type OpenAIResponsesPayloadWithDeepSeekThinking = Omit & { thinking: { type: 'disabled' } }; type OpenAIResponsesPayloadWithQwenThinking = Omit & { enable_thinking: false }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index d10eafe6a0..aee154870a 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -1,6 +1,6 @@ -// Chat Completions as a pipeline — the reference chain for the chat families. +// OpenAI Chat Completions as a pipeline — the reference chain for the chat families. // -// emitOpenAIChatCompletions the edge: writes the answer in the shape the client asked for +// emitOpenAIChatCompletions the edge: writes the answer in the shape the client asked for // writeSettlement above the fork, so a run bills once however many wires it tried // resolveChatCandidates narrows to what can serve, in the order affinity asks for // failover runs what follows once per candidate @@ -14,7 +14,7 @@ // tell which ran. // // What sits *in* a wire rather than above the fork is the other half of the arrangement. A -// rule that speaks about an upstream's Chat Completions endpoint — the role rewrite does, +// rule that speaks about an upstream's OpenAI Chat Completions endpoint — the role rewrite does, // which is why its interceptor form stood down on `ctx.targetApi !== 'openaiChatCompletions'` — // belongs to the wire and not to the source chain, so a turn that leaves for another // protocol never carries it. Said by position rather than by a guard: a stage below the fork @@ -68,8 +68,8 @@ import type { BillableUsage, ProtocolFrame, SseFrame } from '@floway-dev/protoco import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateOpenAIChatCompletionsViaAnthropicMessages, translateOpenAIChatCompletionsViaOpenAIResponses } from '@floway-dev/translate'; -/** `/v1/chat/completions` prefers its own wire, then the translated Messages path, then the - * translated Responses path. */ +/** `/v1/chat/completions` prefers its own wire, then the translated Anthropic Messages path, + * then the translated OpenAI Responses path. */ export const openaiChatCompletionsTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); /** What this family adds to the chat space. */ @@ -188,7 +188,7 @@ const renderSSE = ( }); /** - * The wire. It dials Chat Completions and provides the answer at whichever family's response + * The wire. It dials OpenAI Chat Completions and provides the answer at whichever family's response * key the chain above it reads — which is what makes it interchangeable with a translated * chain: both hand up `response.chat.openaiChatCompletions`, and the stage above cannot tell which * ran. @@ -284,10 +284,10 @@ const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage }); /** - * The Chat Completions wire, as the chain that dials it. + * The OpenAI Chat Completions wire, as the chain that dials it. * * Every source protocol that reaches an upstream over this endpoint runs this, whether the - * client spoke Chat Completions or a handoff arrived here — which is what makes a rule that + * client spoke OpenAI Chat Completions or a handoff arrived here — which is what makes a rule that * speaks about *this* wire belong here. The role rewrite states what an upstream's Chat * Completions endpoint accepts; the usage rules speak about the usage this wire asks for and * this wire reports; the vendor dialects are how one upstream spells both. All of them apply @@ -301,7 +301,7 @@ const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage * seeing usage the fold has already settled. * * `disableReasoningOnForcedToolChoice` and `stripPromptCacheKey` are here rather than above - * the fork, because both speak about what an upstream's Chat Completions endpoint accepts — + * the fork, because both speak about what an upstream's OpenAI Chat Completions endpoint accepts — * so a turn that arrived over a translation gets them too. Both still precede every vendor * dialect on the request path: the canonical sentinel is emitted before a vendor spells it, * and the field an upstream would reject is gone before a vendor rewrites what is left. diff --git a/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts b/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts index c62fdce8a6..88b763ce75 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts @@ -15,7 +15,7 @@ export const openaiResponsesCreatedAt = (ctx: GatewayCtx): number => Math.floor( // The item lifecycle first resolves any partial terminal restatement. Affinity // then wraps routing metadata, and the client-output boundary stores each // complete emitted item under its exact ID and applies one generated response -// ID to the downstream stream and snapshot. Every native Responses turn goes +// ID to the downstream stream and snapshot. Every native OpenAI Responses turn goes // through this half, whichever resource it answers with. export const wrapOpenAIResponsesStatefulOutput = ( frames: AsyncIterable>, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts index 9c04f35e10..e687b627fb 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts @@ -291,7 +291,7 @@ export const summarizationTurnFor = (payload: CanonicalOpenAIResponsesPayload): // A summarization that closed no assistant text produced no summary, and a // compaction blob is the whole of what the next turn inherits — so an empty one // silently discards the conversation. -export const EMPTY_SUMMARY_MESSAGE = 'Responses compact shim: the summarization turn closed no assistant text to summarize'; +export const EMPTY_SUMMARY_MESSAGE = 'OpenAI Responses compact shim: the summarization turn closed no assistant text to summarize'; export const containsCompactionTrigger = (input: readonly OpenAIResponsesInputItem[]): boolean => input.some(item => item.type === 'compaction_trigger'); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index a8e0846015..1cceb0d50f 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -22,8 +22,8 @@ // - the compaction wire. An upstream whose own `/responses` endpoint compacts is asked to, // and answers with one envelope; the synthetic frames it becomes are what the stateful // half above persists items and a response id from. -// - the simulation. An upstream with no compaction wire of its own — every Messages and -// Chat Completions candidate, and any Responses candidate an operator opted in with +// - the simulation. An upstream with no compaction wire of its own — every Anthropic Messages +// and OpenAI Chat Completions candidate, and any OpenAI Responses candidate an operator opted in with // `responses-compact-shim` — is sent an ordinary generate turn against the compactor's // prompt over the wires generation would have used, and the summary it produces is packed // into an envelope of this gateway's own. That is the whole of what the action pivot was: @@ -309,10 +309,10 @@ const simulationWire: ChatWire = compose('openaiResponsesCompactSimulated', [sum * Picks how this candidate's compaction is produced. * * An upstream whose own endpoint compacts is asked to, unless the operator asked for the - * simulation instead — which is what the `responses-compact-shim` flag is: an opt-in for a - * Responses upstream that would answer a compaction itself. Everything else is simulated, - * structurally rather than by choice: no translation carries a compaction, so a Messages or - * Chat Completions candidate has no compaction to dial. + * simulation instead — which is what the `responses-compact-shim` flag is: an opt-in for an + * OpenAI Responses upstream that would answer a compaction itself. Everything else is simulated, + * structurally rather than by choice: no translation carries a compaction, so an Anthropic + * Messages or OpenAI Chat Completions candidate has no compaction to dial. * * It is last, which is what earns it the right to name a target at all, and it holds no state * across candidates: failover re-running the suffix re-runs this stage. diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts index b346b0e0e5..f87f1d6c54 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts @@ -929,7 +929,7 @@ async function* runMultiTurnLoop(args: { } // Accumulated output items are fed back as the next turn's input. - // A OpenAI Responses output item is a structural superset of the matching + // An OpenAI Responses output item is a structural superset of the matching // input item for every shape we emit here (messages, reasoning, // function_call / function_call_output, and the server-tool items // the dispatchers produce), so the reuse is sound; the cast only diff --git a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts index 34f3d1dc17..fdb0a0d5aa 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts @@ -184,7 +184,7 @@ export class LayeredStatefulOpenAIResponsesStore implements StatefulOpenAIRespon if (item.type === 'compaction_trigger') return; if (item.type === 'item_reference') { const row = this.loadedItems.get(item.id); - if (row === undefined) throw new Error(`Cannot stage unresolved Responses item_reference id=${item.id}`); + if (row === undefined) throw new Error(`Cannot stage unresolved OpenAI Responses item_reference id=${item.id}`); this.stagedInputItemIds.push(row.id); return; } @@ -250,7 +250,7 @@ export class LayeredStatefulOpenAIResponsesStore implements StatefulOpenAIRespon const pending = rows.flatMap(row => { if (!this.committedItemIds.has(row.id)) return [row]; const committed = this.loadedItems.get(row.id); - if (committed === undefined) throw new Error(`Committed Responses item disappeared from request state: ${row.id}`); + if (committed === undefined) throw new Error(`Committed OpenAI Responses item disappeared from request state: ${row.id}`); assertSameStoredOpenAIResponsesItem(row, committed); return []; }); @@ -387,8 +387,9 @@ export const createOpenAIResponsesHttpStore = (apiKey: OpenAIResponsesStatePolic }); }; -// Non-Responses sources (Messages / Gemini / Chat Completions) never persist -// Responses items, even when translation enters a Responses attempt — but the +// Non-OpenAI-Responses sources (Anthropic Messages / Gemini generateContent / OpenAI Chat +// Completions) never persist OpenAI Responses items, even when translation enters an OpenAI +// Responses attempt — but the // server-tool shim's request-private payload scratchpad lives on the store, and // it is what will write there once that shim is a stage. So they get a store // with no backing: it holds per-attempt state in memory and reads/writes nothing diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 191d61ad77..ce82b7bf6c 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -1,6 +1,6 @@ -// Responses as a pipeline, on the chain Chat Completions established. +// OpenAI Responses as a pipeline, on the chain OpenAI Chat Completions established. // -// emitOpenAIResponses the edge: writes the answer in the shape the client asked for +// emitOpenAIResponses the edge: writes the answer in the shape the client asked for // writeSettlement above the fork, so a run bills once however many wires it tried // hydrateStoredItems the stored-items membrane, on the way in // resolveChatCandidates narrows to what can serve, in the order affinity asks for @@ -12,7 +12,7 @@ // dialChatWire the ending: picks this candidate's wire and hands into it // // Three wires, all handing up `response.chat.openaiResponses`: this protocol's own, and the two -// translated ones — Responses via Messages and Responses via Chat Completions — each a +// translated ones — OpenAI Responses via Anthropic Messages and via OpenAI Chat Completions — each a // handoff followed by that protocol's own wire. What sits *in* a wire rather than above the // fork is a rule that speaks about that wire: the role rewrite and the cache-bucket fold both // do, which is why their interceptor forms stood down on `ctx.targetApi !== 'openaiResponses'`, and @@ -106,8 +106,8 @@ import { import { providerModelOf, toInternalDebugError, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateOpenAIResponsesViaOpenAIChatCompletions, translateOpenAIResponsesViaAnthropicMessages } from '@floway-dev/translate'; -/** `/v1/responses` prefers its own wire, then the translated Messages path, then the - * translated Chat Completions path. */ +/** `/v1/responses` prefers its own wire, then the translated Anthropic Messages path, then + * the translated OpenAI Chat Completions path. */ export const openaiResponsesTarget = chatTargetPicker(['openaiResponses', 'anthropicMessages', 'openaiChatCompletions']); /** How the transport that opened the run frames a streamed answer. Both carry this @@ -303,7 +303,7 @@ const streamFailedEvent = (announced: ClientResponseResource, error: unknown): C } as ClientOpenAIResponsesStreamEvent; }; -/** Every Responses event has an SSE form of its own, so the render is a straight map. What +/** Every OpenAI Responses event has an SSE form of its own, so the render is a straight map. What * it adds is the two endings a client already being streamed to can be given. The ordinary * one is the terminator: the client's stream ends on the literal `[DONE]` payload whether or * not the upstream's stream carried one, because that is what the transport reads to know @@ -340,7 +340,7 @@ const renderSSE = (frames: AsyncIterable defineStage< }); /** - * What an upstream's Responses endpoint accepts on an assistant item it produced itself. + * What an upstream's OpenAI Responses endpoint accepts on an assistant item it produced itself. * Copilot's compaction translation and Azure-native compaction both emit assistant messages * whose content blocks carry `type: 'input_text'`, and both then refuse those same items * echoed back as input on the next turn. Every way prior upstream-produced history reaches a @@ -461,7 +461,7 @@ const callOpenAIResponsesUpstream = (streamedUsage: string) => defineStage< * tail — so this is the one place the canonical assistant content type is put back. * * Only this wire needs it. Both translators read `input_text` and `output_text` the same way - * on assistant content, so a turn that leaves for Messages or Chat Completions never carried + * on assistant content, so a turn that leaves for Anthropic Messages or OpenAI Chat Completions never carried * the disagreement in the first place. */ const normalizeAssistantContentForOpenAIResponses = defineStage< @@ -490,9 +490,9 @@ const normalizeAssistantContentForOpenAIResponses = defineStage< * What every turn this wire sends is subject to, whichever operation asked for it. * * Every source protocol that reaches an upstream over this endpoint runs these, whether the - * client spoke Responses or a handoff arrived here — which is what makes the three rules + * client spoke OpenAI Responses or a handoff arrived here — which is what makes the three rules * belong to the wire. The role rewrite and the assistant-content rewrite both state what an - * upstream's Responses endpoint accepts; the cache-bucket fold speaks about the usage *this* + * upstream's OpenAI Responses endpoint accepts; the cache-bucket fold speaks about the usage *this* * wire reports and about the flag that describes it, and a translator emits the canonical * form, which is the one case the fold has nothing to do with. * @@ -509,7 +509,7 @@ export const openaiResponsesWireRules: readonly Stage[] = [ vendorQwenNormalizeForOpenAIResponses, ]; -/** The Responses wire, as the chain that dials it. */ +/** The OpenAI Responses wire, as the chain that dials it. */ export const openaiResponsesWire = (streamedUsage: string): readonly Stage[] => [ ...openaiResponsesWireRules, callOpenAIResponsesUpstream(streamedUsage), @@ -550,7 +550,7 @@ export const openaiResponsesWireFor = (target: ChatTargetApi, candidate: ModelCa }; /** Reads the upstream's own usage off its own events as they pass, so the reading costs one - * pass and the client's stream is what drives it. Responses states its counts on the + * pass and the client's stream is what drives it. OpenAI Responses states its counts on the * lifecycle envelopes, and only one carrying real counts replaces the running figure, so an * envelope that states none cannot wipe a good reading. */ const meterOpenAIResponses = ( @@ -799,10 +799,10 @@ export const beginStoredAttempt = defineStage< /** * Whether this candidate's compactions are the shim's to simulate. * - * Two ways to be, and an upstream only has to be one of them. The operator opted a Responses + * Two ways to be, and an upstream only has to be one of them. The operator opted an OpenAI Responses * upstream in with `responses-compact-shim`, which is the say-so for one that would answer a - * compaction itself. Or the candidate has no Responses endpoint at all: no translation - * carries a compaction, so a Messages or Chat Completions candidate has neither a compaction + * compaction itself. Or the candidate has no OpenAI Responses endpoint at all: no translation + * carries a compaction, so an Anthropic Messages or OpenAI Chat Completions candidate has neither a compaction * to dial nor a translator that models the item asking for one — the shim is structurally * required there rather than chosen. * @@ -943,7 +943,7 @@ export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< * with the control item that requests it — so this chain reads the request rather than the * operation, and reads it per turn. Where the shim is not this candidate's to run, the item * travels on to an upstream whose own `/responses` endpoint answers it, which is what a - * native Responses upstream does with it. + * native OpenAI Responses upstream does with it. */ const asksForCompaction: CompactionAsk = (facts, use) => containsCompactionTrigger((facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload).input) diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index 7f1f71a7ca..3c5e821ce2 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -1,4 +1,4 @@ -// The Responses WebSocket transport: a second entry against the same chain `POST +// The OpenAI Responses WebSocket transport: a second entry against the same chain `POST // /v1/responses` runs. One `response.create` frame is one turn and one turn is one run, so // nothing about a turn outlives the frame that opened it except what the connection itself // holds — the session's item store and the ids a continuation may name. @@ -572,7 +572,7 @@ const respondOpenAIResponsesWebSocket = async (input: { // streaming HTTP, and the specification defines no frame after the // terminal event: // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L117 - // Live captures agree. Azure Foundry Responses-over-WebSocket, and the + // Live captures agree. Azure Foundry OpenAI-Responses-over-WebSocket, and the // ChatGPT backend the Codex CLI talks to, both end a turn on // `response.completed` and send nothing further while the socket stays // open and idle. Codex's own reader breaks its loop on that event diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts index 5487c02ab7..1de6b0437b 100644 --- a/packages/gateway/src/data-plane/chat/prologue.ts +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -28,7 +28,7 @@ export const openChatPrologue = ( options: { readonly wantsStream: boolean; readonly model?: string; - /** Native Responses entries persist their items; every other source gets a scratchpad, + /** Native OpenAI Responses entries persist their items; every other source gets a scratchpad, * so the server-tool shim's request-private state always has a home. */ readonly storeFactory: (apiKey: ApiKey, requestStartedAt: number) => StatefulOpenAIResponsesStore; }, diff --git a/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts b/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts index bb481f309f..203035dec9 100644 --- a/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts +++ b/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts @@ -1,6 +1,6 @@ // What a source's stream turned out to be, read back when the turn settles. // -// One transport still needs it: the Responses WebSocket writes each frame itself and has to +// One transport still needs it: the OpenAI Responses WebSocket writes each frame itself and has to // decide, when the socket closes, whether what it wrote was a finished turn. Every HTTP // family answers the same question from its own metered reading instead. diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts index d1ab6a8562..a32c048f55 100644 --- a/packages/gateway/src/data-plane/chat/stages.ts +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -43,7 +43,7 @@ export interface ChatNarrowing { * appear here cannot serve the request whatever else it offers. */ readonly canServe: (candidate: ModelCandidate) => boolean; /** What the client's own turn says about where it must go. Client-carried state — a - * Responses `previous_response_id`, an encrypted reasoning blob — pins the turn to the + * an OpenAI Responses `previous_response_id`, an encrypted reasoning blob — pins the turn to the * upstream that issued it, so this runs before any candidate is tried. */ readonly affinity: (gateway: ChatGatewayCtx) => Promise>; readonly unsupported: (model: string) => string; diff --git a/packages/translate/src/anthropic-messages-via-openai-responses/events.ts b/packages/translate/src/anthropic-messages-via-openai-responses/events.ts index e8a8e2dc28..e1d67b65eb 100644 --- a/packages/translate/src/anthropic-messages-via-openai-responses/events.ts +++ b/packages/translate/src/anthropic-messages-via-openai-responses/events.ts @@ -396,7 +396,7 @@ const handleCompleted = (response: OpenAIResponsesResult, state: OpenAIResponses return events; }; -// A OpenAI Responses upstream can report a context-exceeded failure inside the SSE +// An OpenAI Responses upstream can report a context-exceeded failure inside the SSE // stream (Codex emits `response.failed` with `error.code = // context_length_exceeded`; some Copilot fronts surface a stream `error` // event with the same code). We rewrite those into the same Anthropic From 6286f55395b25a851af501ef5ba59596b052969e Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 06:11:21 +0800 Subject: [PATCH 082/156] refactor: close the last abbreviated protocol references in the chat area The sweep the type checker cannot do: three client-facing sentences the OpenAI Responses store and WebSocket write, the dump failure tag, comments naming renamed files and pickers, the `ChatTargetApi` values quoted in test names, and the two locale strings that still said "Messages API". `ANTHROPIC_ANTHROPIC_MESSAGES_ENDPOINT` loses the vendor the earlier rename doubled onto it: `ANTHROPIC_MESSAGES_ENDPOINT` already named the protocol. --- apps/web/src/i18n/locales/en.ts | 2 +- apps/web/src/i18n/locales/zh-Hans.ts | 2 +- .../chat/anthropic-messages/http_test.ts | 2 +- ...mini-generate-content-count-tokens_test.ts | 2 +- .../gemini-generate-content-pipeline_test.ts | 2 +- .../chat/gemini-generate-content/http_test.ts | 2 +- .../chat/openai-chat-completions/http_test.ts | 2 +- .../openai-responses/compact-shim_test.ts | 4 +-- .../interceptors/server-tool-shim_test.ts | 6 ++-- .../chat/shared/alias-rules_test.ts | 34 +++++++++---------- .../chat/shared/first-output-token_test.ts | 6 ++-- .../chat/openai-responses/compact-shim.ts | 2 +- .../chat/openai-responses/items/store.ts | 4 +-- .../chat/openai-responses/websocket.ts | 4 +-- packages/provider-claude-code/src/fetch.ts | 4 +-- 15 files changed, 39 insertions(+), 39 deletions(-) diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index 464193e501..79892a0192 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -631,7 +631,7 @@ const en = { 'messages-web-search-shim': { label: 'Anthropic Messages Web Search Shim', description: - 'The Anthropic Messages API includes web search capabilities, but this upstream may not support search.\nEnable this option to handle web search tool calls through the search provider configured in Floway instead of forwarding them to the upstream.\nThis option is treated as enabled when the upstream does not provide the Messages API.', + 'The Anthropic Messages API includes web search capabilities, but this upstream may not support search.\nEnable this option to handle web search tool calls through the search provider configured in Floway instead of forwarding them to the upstream.\nThis option is treated as enabled when the upstream does not provide the Anthropic Messages API.', }, 'responses-web-search-shim': { label: 'OpenAI Responses Web Search Shim', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index f3560c0d2d..67df712331 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -601,7 +601,7 @@ const zhHansCN = { 'messages-web-search-shim': { label: 'Anthropic Messages 网页搜索兼容层', description: - 'Anthropic Messages API 包含搜索能力,但本上游可能不支持搜索。\n开启此开关,以通过 Floway 配置的搜索提供商来处理网页搜索工具调用,而非转发到上游。\n当上游不提供 Messages API 时,此开关被视为开启。', + 'Anthropic Messages API 包含搜索能力,但本上游可能不支持搜索。\n开启此开关,以通过 Floway 配置的搜索提供商来处理网页搜索工具调用,而非转发到上游。\n当上游不提供 Anthropic Messages API 时,此开关被视为开启。', }, 'responses-web-search-shim': { label: 'OpenAI Responses 网页搜索兼容层', diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index 1b63323603..2142d821b4 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -305,7 +305,7 @@ test('POST /v1/messages forwards upstream response headers end-to-end (non-strea assertEquals(response.headers.get('cf-ray'), 'cf_ray_e2e'); }); -test('POST /v1/messages renders the Anthropic-shaped model-unsupported 400 when no candidate matches the messages-generate picker', async () => { +test('POST /v1/messages renders the Anthropic-shaped model-unsupported 400 when no candidate matches the anthropic-messages-generate picker', async () => { installRepo(); // Queue a chat-kind candidate whose endpoints expose only `completions` — // anthropicMessagesGenerateTarget (messages > responses > chat-completions) rejects diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts index f3b8768188..d4ac825464 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts @@ -106,7 +106,7 @@ beforeEach(() => { } as never); }); -describe('the gemini count-tokens chain', () => { +describe('the Gemini generateContent count-tokens chain', () => { // Gemini generateContent has no endpoint that answers this, so the question is asked in Anthropic Messages and the // counts are read back out under the name this protocol gives them. it('asks the question in Anthropic Messages and answers in Google-s own envelope', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts index 4fc6a8bf18..b030f6ee6b 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts @@ -185,7 +185,7 @@ const affinityCarriers = (events: readonly GeminiGenerateContentResult[]): reado events.flatMap(event => (event.candidates ?? []).flatMap(candidate => candidate.content.parts.flatMap(part => part.thoughtSignature ?? []))); -describe('the gemini pipeline', () => { +describe('the Gemini generateContent pipeline', () => { // The whole entry contract, and it does not mention `request.chat.geminiGenerateContent`: the turn the // wire translates is not the one the caller handed in but the one materializeAttempt put // into the record below the fork, which is where a per-candidate payload can exist at all. diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts index 19c15dcee0..c150d6669f 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts @@ -102,7 +102,7 @@ const makeCandidate = (overrides: { const targetApi = overrides.targetApi ?? 'openaiChatCompletions'; // When the test fixes `endpoints` directly, use it verbatim — that lets a // test pin a wrong-endpoint shape the picker rejects. Otherwise synthesize - // a single-endpoint map from `targetApi` so the gemini serve layer's + // a single-endpoint map from `targetApi` so the Gemini generateContent serve layer's // picker (openai-chat-completions first, then messages, then responses) lands on // the requested wire. const endpoints = overrides.endpoints ?? (targetApi === 'openaiChatCompletions' diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts index 3df6136d2d..7933d53713 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts @@ -355,7 +355,7 @@ test('POST /v1/chat/completions renders the OpenAI-shaped model-unsupported 400 installRepo(); // Queue a chat-kind candidate whose endpoints expose only `openaiCompletions` — // the openaiChatCompletionsTarget picker rejects it (its preference list is - // `openai-chat-completions` > `messages` > `responses`), leaving zero viable + // `openaiChatCompletions` > `anthropicMessages` > `openaiResponses`), leaving zero viable // candidates, and with sawModel=true the serve renders model-unsupported // as a 400. queueCandidates([makeCandidate({ endpoints: { openaiCompletions: {} } })]); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts index 42cb1f1d48..aafe72cb13 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts @@ -1,6 +1,6 @@ // What a simulated compaction is made of. The rules that use these run as stages and are -// tested where they run — `responses-pipeline_test.ts` for the generate chain and -// `responses-compact_test.ts` for the compaction chain; what is written down here is the +// tested where they run — `openai-responses-pipeline_test.ts` for the generate chain and +// `openai-responses-compact_test.ts` for the compaction chain; what is written down here is the // substance those stages send and pack. import { test } from 'vitest'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts index 85c1a1904e..b86b23fe24 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts @@ -444,7 +444,7 @@ const outputItemDoneEvents = (frames: ProtocolFrame[ // ── Activation gating ────────────────────────────────────────────────── -test('shim no-ops when targetApi=responses and flag is off', async () => { +test('shim no-ops when targetApi=openaiResponses and flag is off', async () => { const { backend } = makeStubDeps(); const shim = withOpenAIResponsesWebSearchShim; const inv = makeInvocation({ @@ -463,7 +463,7 @@ test('shim no-ops when targetApi=responses and flag is off', async () => { assertEquals(result.type, 'events'); }); -test('shim activates when targetApi=responses and flag is on', async () => { +test('shim activates when targetApi=openaiResponses and flag is on', async () => { makeStubDeps(); const shim = withOpenAIResponsesWebSearchShim; const inv = makeInvocation({ @@ -479,7 +479,7 @@ test('shim activates when targetApi=responses and flag is on', async () => { assertEquals((inv.payload.tools?.[0] as { name: string }).name, SHIM_TOOL_NAME); }); -test('shim activates when targetApi=messages and flag is off', async () => { +test('shim activates when targetApi=anthropicMessages and flag is off', async () => { makeStubDeps(); const shim = withOpenAIResponsesWebSearchShim; const inv = makeInvocation({ targetApi: 'anthropicMessages', enabledFlags: new Set() }); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts index f74dfe9e33..6245758136 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts @@ -80,7 +80,7 @@ test('openai-chat-completions: alias rules overwrite existing IR fields', () => // ── OpenAI Responses target ── -test('responses: empty rules leave the payload unchanged', () => { +test('openai-responses: empty rules leave the payload unchanged', () => { const body = resPayload({ reasoning: { effort: 'high' }, text: { verbosity: 'low' }, service_tier: 'priority' }); applyRulesToUpstreamOpenAIResponses(body, {}); assertEquals(body.reasoning?.effort, 'high'); @@ -88,7 +88,7 @@ test('responses: empty rules leave the payload unchanged', () => { assertEquals(body.service_tier, 'priority'); }); -test('responses: rules stamp every supported native field onto the IR', () => { +test('openai-responses: rules stamp every supported native field onto the IR', () => { const body = resPayload(); applyRulesToUpstreamOpenAIResponses(body, { reasoning: { effort: 'high', summary: 'concise' }, @@ -101,7 +101,7 @@ test('responses: rules stamp every supported native field onto the IR', () => { assertEquals(body.service_tier, 'flex'); }); -test('responses: budget_tokens / adaptive have no native slot — silently dropped', () => { +test('openai-responses: budget_tokens / adaptive have no native slot — silently dropped', () => { const body = resPayload(); applyRulesToUpstreamOpenAIResponses(body, { reasoning: { budget_tokens: 1024, adaptive: true }, @@ -111,7 +111,7 @@ test('responses: budget_tokens / adaptive have no native slot — silently dropp assertEquals('adaptive_thinking' in body, false); }); -test('responses: alias rules overwrite owned fields while preserving reasoning.context', () => { +test('openai-responses: alias rules overwrite owned fields while preserving reasoning.context', () => { const body = resPayload({ reasoning: { effort: 'low', summary: 'auto', context: 'future_mode' }, service_tier: 'default', @@ -129,7 +129,7 @@ test('responses: alias rules overwrite owned fields while preserving reasoning.c // ── Anthropic Messages target ── -test('messages: empty rules leave the payload unchanged', () => { +test('anthropic-messages: empty rules leave the payload unchanged', () => { const body = msgPayload({ output_config: { effort: 'high' }, thinking: { type: 'enabled', budget_tokens: 512 }, speed: 'fast' }); applyRulesToUpstreamAnthropicMessages(body, {}); assertEquals(body.output_config?.effort, 'high'); @@ -137,7 +137,7 @@ test('messages: empty rules leave the payload unchanged', () => { assertEquals(body.speed, 'fast'); }); -test('messages: effort lands on output_config, budget+adaptive land on thinking', () => { +test('anthropic-messages: effort lands on output_config, budget+adaptive land on thinking', () => { const body = msgPayload(); applyRulesToUpstreamAnthropicMessages(body, { reasoning: { effort: 'high', budget_tokens: 2048 }, @@ -147,38 +147,38 @@ test('messages: effort lands on output_config, budget+adaptive land on thinking' assertEquals(body.thinking?.budget_tokens, 2048); }); -test('messages: verbosity has no Anthropic-shaped slot — silently dropped', () => { +test('anthropic-messages: verbosity has no Anthropic-shaped slot — silently dropped', () => { const body = msgPayload(); applyRulesToUpstreamAnthropicMessages(body, { verbosity: 'low' }); assertEquals('verbosity' in body, false); }); -test('messages: summary=concise|detailed collapses onto thinking.display=summarized (enables thinking)', () => { +test('anthropic-messages: summary=concise|detailed collapses onto thinking.display=summarized (enables thinking)', () => { const body = msgPayload(); applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'concise' } }); assertEquals(body.thinking?.type, 'enabled'); assertEquals(body.thinking?.display, 'summarized'); }); -test('messages: summary=omitted collapses onto thinking.display=omitted', () => { +test('anthropic-messages: summary=omitted collapses onto thinking.display=omitted', () => { const body = msgPayload(); applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'omitted' } }); assertEquals(body.thinking?.display, 'omitted'); }); -test('messages: summary=auto is a no-op (Anthropic default takes over)', () => { +test('anthropic-messages: summary=auto is a no-op (Anthropic default takes over)', () => { const body = msgPayload(); applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'auto' } }); assertEquals(body.thinking, undefined); }); -test('messages: adaptive=true sets thinking.type=adaptive and ignores budget_tokens', () => { +test('anthropic-messages: adaptive=true sets thinking.type=adaptive and ignores budget_tokens', () => { const body = msgPayload(); applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true, budget_tokens: 4096 } }); assertEquals(body.thinking?.type, 'adaptive'); }); -test('messages: adaptive=true strips a client-set budget_tokens from body.thinking', () => { +test('anthropic-messages: adaptive=true strips a client-set budget_tokens from body.thinking', () => { const body = msgPayload(); body.thinking = { type: 'enabled', budget_tokens: 5000 }; applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true } }); @@ -189,21 +189,21 @@ test('messages: adaptive=true strips a client-set budget_tokens from body.thinki assertEquals((body.thinking as { budget_tokens?: number }).budget_tokens, undefined); }); -test('messages: serviceTier=fast maps to speed=fast (cross-protocol bridge)', () => { +test('anthropic-messages: serviceTier=fast maps to speed=fast (cross-protocol bridge)', () => { const body = msgPayload(); applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); assertEquals(body.speed, 'fast'); assertEquals(body.service_tier, undefined); }); -test('messages: non-fast serviceTier lands on service_tier directly', () => { +test('anthropic-messages: non-fast serviceTier lands on service_tier directly', () => { const body = msgPayload(); applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'priority' }); assertEquals(body.service_tier, 'priority'); assertEquals(body.speed, undefined); }); -test('messages: serviceTier=fast clears a pre-existing body.service_tier on the same payload', () => { +test('anthropic-messages: serviceTier=fast clears a pre-existing body.service_tier on the same payload', () => { // Upstream must never see both `speed` and `service_tier` set on the // same request — Anthropic treats them as alternates and the wire // semantics for a conflict are undefined. The overlay clears the @@ -214,14 +214,14 @@ test('messages: serviceTier=fast clears a pre-existing body.service_tier on the assertEquals(body.service_tier, undefined); }); -test('messages: non-fast serviceTier clears a pre-existing body.speed on the same payload', () => { +test('anthropic-messages: non-fast serviceTier clears a pre-existing body.speed on the same payload', () => { const body = msgPayload({ speed: 'fast' }); applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'priority' }); assertEquals(body.service_tier, 'priority'); assertEquals(body.speed, undefined); }); -test('messages: alias rules overwrite existing thinking + output_config fields', () => { +test('anthropic-messages: alias rules overwrite existing thinking + output_config fields', () => { const body = msgPayload({ output_config: { effort: 'low' }, thinking: { type: 'enabled', budget_tokens: 100 } }); applyRulesToUpstreamAnthropicMessages(body, { reasoning: { effort: 'xhigh', budget_tokens: 9999 } }); assertEquals(body.output_config?.effort, 'xhigh'); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/first-output-token_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/first-output-token_test.ts index 4ecbfd27a0..99c888b92e 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/first-output-token_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/first-output-token_test.ts @@ -5,7 +5,7 @@ import type { ProtocolFrame } from '@floway-dev/protocols/common'; const eventFrame = (event: T): ProtocolFrame => ({ type: 'event', event }); -describe('isFirstOutputTokenFrame — messages', () => { +describe('isFirstOutputTokenFrame — anthropicMessages', () => { it('accepts text_delta', () => { expect(isFirstOutputTokenFrame(eventFrame({ type: 'content_block_delta', delta: { type: 'text_delta', text: 'hi' } }), 'anthropicMessages')).toBe(true); }); @@ -35,7 +35,7 @@ describe('isFirstOutputTokenFrame — messages', () => { }); }); -describe('isFirstOutputTokenFrame — responses', () => { +describe('isFirstOutputTokenFrame — openaiResponses', () => { it('accepts response.output_text.delta', () => { expect(isFirstOutputTokenFrame(eventFrame({ type: 'response.output_text.delta', delta: 'hi' }), 'openaiResponses')).toBe(true); }); @@ -68,7 +68,7 @@ describe('isFirstOutputTokenFrame — responses', () => { }); }); -describe('isFirstOutputTokenFrame — openai-chat-completions', () => { +describe('isFirstOutputTokenFrame — openaiChatCompletions', () => { it('accepts chunk with delta.content', () => { expect(isFirstOutputTokenFrame(eventFrame({ choices: [{ delta: { content: 'hi' } }] }), 'openaiChatCompletions')).toBe(true); }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts index e687b627fb..3cbbadcb1a 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts @@ -4,7 +4,7 @@ // The rules that use these are stages: `simulatesCompaction` says which candidates' // compactions are the shim's to make, `expandShimCompactions` puts a blob of ours back into // the items it stood for, and `summarizeForCompaction` sends the turn built here and packs -// what comes back (`responses/pipeline.ts`). Both of this protocol's chains compose them, +// what comes back (`openai-responses/pipeline.ts`). Both of this protocol's chains compose them, // because a compaction the shim wrote is issued through one entry — `/v1/responses/compact`, // or a generate turn whose input ends in a `compaction_trigger` — and echoed back into the // other by the client that received it. diff --git a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts index fdb0a0d5aa..d4da3e389b 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts @@ -135,7 +135,7 @@ export class LayeredStatefulOpenAIResponsesStore implements StatefulOpenAIRespon if (itemIds.length === 0) return; const uniqueRows = [...new Set(itemIds)].map(id => { const row = this.loadedItems.get(id); - if (row === undefined) throw new Error(`Responses snapshot item disappeared before commit: ${id}`); + if (row === undefined) throw new Error(`OpenAI Responses snapshot item disappeared before commit: ${id}`); return row; }); await this.commitItems(uniqueRows); @@ -335,7 +335,7 @@ export class MemoryStatefulOpenAIResponsesBacking implements StatefulOpenAIRespo const existing = items.map(item => this.items.get(scopedOpenAIResponsesKey(item.apiKeyId, item.id))); const missingIndex = existing.findIndex(item => item === undefined); if (missingIndex !== -1) { - throw new Error(`Responses item disappeared before retention refresh: ${items[missingIndex].id}`); + throw new Error(`OpenAI Responses item disappeared before retention refresh: ${items[missingIndex].id}`); } for (let index = 0; index < existing.length; index += 1) { assertSameStoredOpenAIResponsesItem(items[index], existing[index]!); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index 3c5e821ce2..1f37cc2ee2 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -638,7 +638,7 @@ const respondOpenAIResponsesWebSocket = async (input: { // nothing at all. Both settle here. For the abandoned turn the eviction is inert — the // connection-local cache dies with the socket. turnFailure.evict(); - ctx.dump?.failed(`responses ws turn failed (completion=${completion}, source-failed=${state.failed})`); + ctx.dump?.failed(`openai-responses ws turn failed (completion=${completion}, source-failed=${state.failed})`); } ctx.dump?.finalize(failed ? 500 : 200, 0); } @@ -747,7 +747,7 @@ const normalizeErrorBody = (body: unknown, status: number): Record= 500 ? 'server_error' : 'invalid_request_error'; const message = typeof source.message === 'string' ? source.message - : `Responses request failed with status ${status}.`; + : `OpenAI Responses request failed with status ${status}.`; return { ...source, type, diff --git a/packages/provider-claude-code/src/fetch.ts b/packages/provider-claude-code/src/fetch.ts index dc952c909a..0f73f1e962 100644 --- a/packages/provider-claude-code/src/fetch.ts +++ b/packages/provider-claude-code/src/fetch.ts @@ -21,7 +21,7 @@ import { type ProviderStreamResult, } from '@floway-dev/provider'; -const ANTHROPIC_ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages?beta=true'; +const ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages?beta=true'; export interface CallClaudeCodeAnthropicMessagesOptions { upstreamId: string; @@ -387,7 +387,7 @@ const performUpstreamCall = async ( // and the real Claude Code client always sets `stream: true`. const wireBody: AnthropicMessagesPayload = { ...opts.body, model: upstreamModelId, stream: true }; - const upstreamFetch = opts.call.wrapUpstreamCall(() => opts.call.fetcher(ANTHROPIC_ANTHROPIC_MESSAGES_ENDPOINT, { + const upstreamFetch = opts.call.wrapUpstreamCall(() => opts.call.fetcher(ANTHROPIC_MESSAGES_ENDPOINT, { method: 'POST', headers, body: jsonRequestBody(wireBody), From 0e949a97f2df5071fa2a57c294cdd28892957166 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 06:20:32 +0800 Subject: [PATCH 083/156] style: reorder the imports the renamed specifiers moved Pure `eslint --fix` output over the chat chains: the longer specifiers sort differently. --- .../__tests__/data-plane/chat-interceptors_test.ts | 4 ++-- .../chat/anthropic-messages-count-tokens_test.ts | 2 +- .../data-plane/chat/anthropic-messages-pipeline_test.ts | 2 +- .../data-plane/chat/anthropic-messages/http_test.ts | 2 +- .../chat/gemini-generate-content-count-tokens_test.ts | 2 +- .../chat/gemini-generate-content-pipeline_test.ts | 2 +- .../chat/openai-chat-completions-pipeline_test.ts | 2 +- .../data-plane/chat/openai-responses-compact_test.ts | 2 +- .../data-plane/chat/openai-responses-pipeline_test.ts | 2 +- .../__tests__/data-plane/chat/translated-wires_test.ts | 8 ++++---- .../src/data-plane/chat/anthropic-messages/http.ts | 2 +- .../interceptors/answer-claude-code-probe.ts | 2 +- .../chat/anthropic-messages/interceptors/types.ts | 2 +- .../src/data-plane/chat/anthropic-messages/pipeline.ts | 4 ++-- packages/gateway/src/data-plane/chat/facts.ts | 4 ++-- .../chat/gemini-generate-content/count-tokens.ts | 2 +- .../src/data-plane/chat/gemini-generate-content/http.ts | 2 +- .../data-plane/chat/gemini-generate-content/pipeline.ts | 4 ++-- packages/gateway/src/data-plane/chat/interceptors.ts | 6 +++--- .../src/data-plane/chat/openai-chat-completions/http.ts | 2 +- .../data-plane/chat/openai-chat-completions/pipeline.ts | 4 ++-- .../src/data-plane/chat/openai-responses/pipeline.ts | 4 ++-- 22 files changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts index 7c85d4eaef..279aed3eca 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts @@ -33,10 +33,10 @@ import { } from '../../src/data-plane/chat/interceptors.ts'; import type { AttemptSelector } from '../../src/data-plane/pipeline/facts.ts'; import { compose, defineStage, move, run } from '@floway-dev/pipeline'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; /** What an attempt is, to a stage that only reads flags. The selector carries them as data, diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts index 09416c883d..9d839622e4 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts @@ -11,8 +11,8 @@ import { enumerateModelCandidates } from '../../../src/data-plane/providers/reso import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import type { AliasRules, ModelEndpoints } from '@floway-dev/protocols/common'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { AliasRules, ModelEndpoints } from '@floway-dev/protocols/common'; import { type FlagId, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts index ea61beafdf..b090abf998 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts @@ -13,8 +13,8 @@ import { enumerateModelCandidates } from '../../../src/data-plane/providers/reso import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import type { SseFrame } from '@floway-dev/protocols/common'; import { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { SseFrame } from '@floway-dev/protocols/common'; import { directFetcher, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index 2142d821b4..0be0b6839f 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -6,8 +6,8 @@ import { initRepo } from '../../../../src/repo/index.ts'; import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; import { flushBackground } from '../../../test-utils/background-tracker.ts'; -import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts index d4ac825464..b533537b92 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts @@ -10,9 +10,9 @@ import { enumerateModelCandidates } from '../../../src/data-plane/providers/reso import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { AnthropicMessagesUpstreamCallOptions, ModelCandidate, ProviderCallResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts index b030f6ee6b..0f2844d1dd 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts @@ -11,9 +11,9 @@ import { enumerateModelCandidates } from '../../../src/data-plane/providers/reso import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, type GeminiGenerateContentPayload, type GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { directFetcher, type ModelCandidate, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts index c3dfc6e74c..e863b53eff 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts @@ -11,8 +11,8 @@ import { enumerateModelCandidates } from '../../../src/data-plane/providers/reso import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import { OPENAI_CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { SseFrame } from '@floway-dev/protocols/common'; +import { OPENAI_CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts index a925a06ed5..684e22ab62 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts @@ -16,8 +16,8 @@ import { initRepo } from '../../../src/repo/index.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts index 16b31d1077..3f35e881e1 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -18,8 +18,8 @@ import { initRepo } from '../../../src/repo/index.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; -import type { ModelEndpoints, SseFrame } from '@floway-dev/protocols/common'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ModelEndpoints, SseFrame } from '@floway-dev/protocols/common'; import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesCompactionResult, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 36125ef410..e038579132 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -14,19 +14,19 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; +import { anthropicMessagesServePipeline, anthropicMessagesWire } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; import { handOff } from '../../../src/data-plane/chat/handoff.ts'; -import { anthropicMessagesServePipeline, anthropicMessagesWire } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; +import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { compose, move, run, type Pipeline } from '@floway-dev/pipeline'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsResult, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import { PROMPT_TOO_LONG_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesResult, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocols/common'; import type { GeminiGenerateContentPayload, GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; -import { PROMPT_TOO_LONG_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesResult, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsResult, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts index f32422ce3e..4c88e54da4 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts @@ -12,8 +12,8 @@ import { anthropicMessagesKeepAlive, anthropicMessagesServePipeline } from './pi import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; -import { openChatPrologue } from '../prologue.ts'; import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; +import { openChatPrologue } from '../prologue.ts'; import { move } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts index 73932e59b8..05fc86ecb7 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts @@ -1,7 +1,7 @@ import type { AnthropicMessagesInterceptor } from './types.ts'; import { telemetryModelIdentity } from '../../../shared/telemetry/attribution.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { generateAnthropicId, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { eventResult, providerModelOf, type ExecuteResult, type ModelCandidate } from '@floway-dev/provider'; // Claude Code answers "is this model usable?" by generating one token against diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts index 44753e3fc5..418b6faadf 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts @@ -1,7 +1,7 @@ import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult, AnthropicMessagesInvocation } from '@floway-dev/provider'; export type { AnthropicMessagesInvocation }; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index ab1ec9c5e2..24876e673d 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -36,7 +36,6 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; -import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { @@ -44,6 +43,7 @@ import { disableReasoningOnForcedToolChoiceForAnthropicMessages, stripBillingAttributionFromAnthropicMessages, } from '../interceptors.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; @@ -51,7 +51,6 @@ import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage } from '@floway-dev/pipeline'; -import { sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { collectAnthropicMessagesProtocolEventsToResult, anthropicMessagesProtocolFrameToSSEFrame, @@ -60,6 +59,7 @@ import { type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent, } from '@floway-dev/protocols/anthropic-messages'; +import { sseFrame, type BillableUsage, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateAnthropicMessagesViaOpenAIChatCompletions, translateAnthropicMessagesViaOpenAIResponses } from '@floway-dev/translate'; diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts index 23ec8bf417..9eb2df7e6c 100644 --- a/packages/gateway/src/data-plane/chat/facts.ts +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -11,9 +11,9 @@ // `wantsStream` is four disjoint keys rather than one shared one. import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; /** The four protocols a client can speak here. Gemini generateContent is source-only: nothing translates diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index a4cc3bfec9..febd5099c6 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -35,12 +35,12 @@ import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import { anthropicMessagesCountTokensWire, type TokenCountAnswer } from '../anthropic-messages/count-tokens.ts'; import { stripSafetySettingsFromGeminiGenerateContent, stripUnsupportedPartFieldsFromGeminiGenerateContent, stripUnsupportedToolsFromGeminiGenerateContent, } from '../interceptors.ts'; -import { anthropicMessagesCountTokensWire, type TokenCountAnswer } from '../anthropic-messages/count-tokens.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts index d0612a9147..897f6ec521 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts @@ -12,8 +12,8 @@ import { geminiGenerateContentServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; -import { openChatPrologue } from '../prologue.ts'; import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; +import { openChatPrologue } from '../prologue.ts'; import { move } from '@floway-dev/pipeline'; import type { GeminiGenerateContentContent, GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index d42e547bec..a623fddfa5 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -57,7 +57,7 @@ import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; -import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { @@ -66,7 +66,7 @@ import { stripUnsupportedToolsFromGeminiGenerateContent, suppressThoughtPartsFromGeminiGenerateContent, } from '../interceptors.ts'; -import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/interceptors.ts index 17b4e4fa67..e4044c6edb 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/interceptors.ts @@ -24,15 +24,15 @@ import { asJsonObject, type JsonObject, readJsonNumber } from '../../shared/json import { isFailure, type AttemptSelector } from '../pipeline/facts.ts'; import { foldsExclusiveCacheTokens } from '../shared/telemetry/usage.ts'; import { defineStage, move, transform } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentContent, GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent, GeminiGenerateContentToolGroup } from '@floway-dev/protocols/gemini-generate-content'; import type { OpenAIChatCompletionsMessage, OpenAIChatCompletionsPayload, OpenAIChatCompletionsReasoningItem, OpenAIChatCompletionsStreamEvent, } from '@floway-dev/protocols/openai-chat-completions'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentContent, GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent, GeminiGenerateContentToolGroup } from '@floway-dev/protocols/gemini-generate-content'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; /** Removing a field inside a value is the same move as removing a fact, and a value that diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts index c7e91d87f5..c272827cbf 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts @@ -13,8 +13,8 @@ import { openaiChatCompletionsServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; -import { openChatPrologue } from '../prologue.ts'; import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; +import { openChatPrologue } from '../prologue.ts'; import { move } from '@floway-dev/pipeline'; import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index aee154870a..33432b0982 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -34,6 +34,7 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { @@ -47,7 +48,6 @@ import { vendorKimiNormalizeForOpenAIChatCompletions, vendorQwenNormalizeForOpenAIChatCompletions, } from '../interceptors.ts'; -import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamOpenAIChatCompletions } from '../shared/alias-rules.ts'; @@ -56,6 +56,7 @@ import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; import { compose, defineStage, move, type Pipeline, type Stage, type Use } from '@floway-dev/pipeline'; +import type { BillableUsage, ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { OPENAI_CHAT_COMPLETIONS_MISSING_TERMINAL_MESSAGE, openaiChatCompletionsErrorPayloadMessage, @@ -64,7 +65,6 @@ import { type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent, } from '@floway-dev/protocols/openai-chat-completions'; -import type { BillableUsage, ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; import { providerModelOf, type ChatTargetApi, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; import { translateOpenAIChatCompletionsViaAnthropicMessages, translateOpenAIChatCompletionsViaOpenAIResponses } from '@floway-dev/translate'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index ce82b7bf6c..f0b4831878 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -71,7 +71,7 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; -import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; import { @@ -82,7 +82,7 @@ import { vendorDeepSeekNormalizeForOpenAIResponses, vendorQwenNormalizeForOpenAIResponses, } from '../interceptors.ts'; -import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import { applyRulesToUpstreamOpenAIResponses } from '../shared/alias-rules.ts'; import { tryCatchChatServeFailure } from '../shared/errors.ts'; import { createExternalImageLoader } from '../shared/external-image-loader.ts'; From 7d5a4993be7ffbaf45ee7fbdd6c23ffdee7b2306 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 06:21:49 +0800 Subject: [PATCH 084/156] docs(openai-responses): name the payload type the projection takes The comment still said `ResponsesPayload`; the type has been `OpenAIResponsesPayload` since the rename. --- packages/protocols/src/openai-responses/compact.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/protocols/src/openai-responses/compact.ts b/packages/protocols/src/openai-responses/compact.ts index 463e5b4f65..482ab134cc 100644 --- a/packages/protocols/src/openai-responses/compact.ts +++ b/packages/protocols/src/openai-responses/compact.ts @@ -34,7 +34,7 @@ export type CanonicalOpenAIResponsesCompactPayload = Omit Date: Wed, 19 Aug 2026 06:33:27 +0800 Subject: [PATCH 085/156] refactor: name the vendor in the last four unqualified protocol comments Four comments outside the chat chains still said "Responses" and "Messages" bare: the search chain, the attempt-body boundary, the OpenAI Images stream reader, and the Copilot boundary chain. --- packages/gateway/src/data-plane/alpha-search/pipeline.ts | 4 ++-- packages/gateway/src/data-plane/pipeline/attempt-body.ts | 2 +- packages/protocols/src/openai-images/stream.ts | 2 +- .../src/interceptors/anthropic-messages/index.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts index 2a2e52893b..fc53535f4a 100644 --- a/packages/gateway/src/data-plane/alpha-search/pipeline.ts +++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts @@ -188,8 +188,8 @@ const executeSearchOperations = defineStage< filters: facts['request.search.filters'], apiKeyId: use.gateway.apiKeyId, pageCache: new Map(), - // Codex renders `output` as plain text; the search-action sources list is a Responses - // protocol concern with no place here. + // Codex renders `output` as plain text; the search-action sources list is an OpenAI + // Responses protocol concern with no place here. includeSearchActionSources: false, ...(use.gateway.abortSignal === undefined ? {} : { signal: use.gateway.abortSignal }), }; diff --git a/packages/gateway/src/data-plane/pipeline/attempt-body.ts b/packages/gateway/src/data-plane/pipeline/attempt-body.ts index 9d750fa904..47633a4008 100644 --- a/packages/gateway/src/data-plane/pipeline/attempt-body.ts +++ b/packages/gateway/src/data-plane/pipeline/attempt-body.ts @@ -4,7 +4,7 @@ // values are frozen — a fact that could be edited after the event is not a fact — while a // provider's interceptors shape the body they are given in place, down to nested nodes: // Copilot writes `copilot_cache_control` onto individual messages, and its initiator rule -// writes into a nested field of a Messages payload. +// writes into a nested field of an Anthropic Messages payload. // // Spreading a frozen fact produces a fresh top level over frozen children, so those writes // throw — as a 502 carrying `Cannot add property …, object is not extensible`, raised at the diff --git a/packages/protocols/src/openai-images/stream.ts b/packages/protocols/src/openai-images/stream.ts index 0596177163..7f8d1ed002 100644 --- a/packages/protocols/src/openai-images/stream.ts +++ b/packages/protocols/src/openai-images/stream.ts @@ -40,7 +40,7 @@ export const parseOpenAIImagesStream = ( /** The event's own name, which the specification writes twice: as the SSE `event:` label and * as `type` inside the payload. Upstreams that write only the label have been seen on the - * Responses protocol, and every reader here takes the name off the payload, so the two are + * OpenAI Responses protocol, and every reader here takes the name off the payload, so the two are * reconciled once at the boundary rather than at each reader. An event that carries neither * is not one this protocol can place — it could be the terminal or a partial — so it ends the * read rather than being passed on as an unknown. */ diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts index 3467971f90..9a8bc5c8a5 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts @@ -1,4 +1,4 @@ -// Copilot-only Messages workarounds. Each list is a boundary chain the +// Copilot-only Anthropic Messages workarounds. Each list is a boundary chain the // Copilot provider runs inside its own `callX` methods, so the gateway main // flow never knows that Copilot has interceptors at all. @@ -23,7 +23,7 @@ import type { CopilotAnthropicMessagesBoundaryInterceptor, CopilotAnthropicMessa // payload): // - rewriteContextWindowError wraps the whole chain so any upstream context- // window failure surfaced from the terminal is rewritten into a -// Messages-shaped invalid_request_error before later interceptors see it. +// Anthropic-Messages-shaped invalid_request_error before later interceptors see it. // - withCompactHeadersSet pins the compact/auto-continue intent first. // - withClaudeAgentHeadersSet then overrides those intents (and the // user-agent / copilot-integration-id) for Claude Code SDK proxy traffic. From 701ffa0993b25463e56dc3cb5396facb947cbf48 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 06:40:14 +0800 Subject: [PATCH 086/156] refactor(copilot): shape a payload by rebuilding it, and stop cloning at the dial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The record is immutable, so nothing in the pipeline should ever need a copy — and it did not, except that three Copilot rules still wrote into the payload they were handed instead of returning a new one. Every other rule in every provider already rebuilt at the top level; these three reached one level further, into a message, a content block, a tool. Against a frozen record that throws, which is what made every Copilot chat turn answer 502. `bodyForAttempt` had been given a `structuredClone` to absorb that. It was the wrong side of the boundary: it paid for a copy of every payload on every dial to protect three lines, and it made the record's immutability something the gateway worked around rather than something the whole system held to. So the three rebuild instead, identity-preserving — an untouched message rides through as the same object, and only the path that changed is new, which is the rule the record itself is folded by. The clone is gone. The three alias overlays were the same shape one file over: `void` helpers that wrote into the body they were handed, safe only because `bodyForAttempt` had happened to rebuild the top level for them. Nothing stated that, so the next edit to either side would have reintroduced the failure. They return the payload they produced now, generic in it so a caller keeps its own narrowing. Two tests asserted the clone's behaviour, that a provider could write into the body it was handed. They now assert the opposite and catch the write, because the frozen record is the guarantee and a rule that reaches into a child must fail loudly rather than be absorbed. Three more pin each overlay against a frozen payload, which is what it is actually handed at the dial. Verified by mutation, on both halves: reverting one rebuild to an in-place write turns every Copilot chat-completions route row red with the original 502, and reverting one overlay fails its frozen case with `Cannot assign to read only property`. --- .../chat/openai-responses-compact_test.ts | 10 +- .../chat/openai-responses-pipeline_test.ts | 10 +- .../chat/shared/alias-rules_test.ts | 186 +++++++++++------- .../data-plane/pipeline/attempt-body_test.ts | 47 +++-- .../src/data-plane/chat/shared/alias-rules.ts | 145 +++++++------- .../src/data-plane/pipeline/attempt-body.ts | 36 ++-- .../apply-top-level-cache-control.ts | 59 ++++-- .../strip-cache-control-extensions.ts | 46 ++--- .../attach-cache-control-markers.ts | 19 +- .../src/interceptors/shared/rebuild.ts | 38 ++++ 10 files changed, 369 insertions(+), 227 deletions(-) create mode 100644 packages/provider-copilot/src/interceptors/shared/rebuild.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts index 684e22ab62..0b8aa0d8c6 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts @@ -394,11 +394,16 @@ describe('the responses compaction chain', () => { // Copilot marks individual items for caching — while the record that body is built from is // deep-frozen. A dial that handed over a shallow copy throws at the first nested write, and // this operation would answer 502 where the upstream had said nothing at all. - it('hands the provider a body it can write into, children included', async () => { + // The record is frozen all the way down, and nothing copies it on the way out — a provider + // shapes the body it sends by rebuilding, which every boundary rule here does. A rule that + // reached into a child instead would throw, and this is what says so rather than leaving it + // to be discovered as a 502 from the one place that cannot explain it. + it('hands the provider a body no one can write into', async () => { + let refused: unknown; resolves([candidate({ callOpenAIResponses: async (model, body, action) => { const item = (body as { input: Record[] }).input[0]!; - item.copilot_cache_control = { type: 'ephemeral' }; + try { item.copilot_cache_control = { type: 'ephemeral' }; } catch (error) { refused = error; } return await compacts({})(model, body, action); }, })]); @@ -406,5 +411,6 @@ describe('the responses compaction chain', () => { const { facts } = await compact(); expect(facts['response.http.status']).toBe(200); + expect(refused).toBeInstanceOf(TypeError); }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts index 3f35e881e1..622bc00dbe 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -544,11 +544,16 @@ describe('the responses chain', () => { // field — while the record that body is built from is deep-frozen. A dial that handed over a // shallow copy satisfies every type in the system and throws at the first nested write, // answering the client with a 502 raised where nothing can explain it. - it('hands the provider a body it can write into, children included', async () => { + // The record is frozen all the way down, and nothing copies it on the way out — a provider + // shapes the body it sends by rebuilding, which every boundary rule here does. A rule that + // reached into a child instead would throw, and this is what says so rather than leaving it + // to be discovered as a 502 from the one place that cannot explain it. + it('hands the provider a body no one can write into', async () => { + let refused: unknown; resolves([candidate({ callOpenAIResponses: async (_model, body) => { const item = (body as { input: Record[] }).input[0]!; - item.copilot_cache_control = { type: 'ephemeral' }; + try { item.copilot_cache_control = { type: 'ephemeral' }; } catch (error) { refused = error; } return stream(completed('hi')); }, })]); @@ -556,6 +561,7 @@ describe('the responses chain', () => { const { facts } = await serve(false); expect(facts['response.http.status']).toBe(200); + expect(refused).toBeInstanceOf(TypeError); }); // Codex asks for a compaction inside an ordinary turn, by ending its input with the control diff --git a/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts index 6245758136..390dca0b76 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts @@ -3,6 +3,9 @@ // are authoritative — an existing IR field is OVERWRITTEN by a matching // rule — and rules the target IR cannot express are silently dropped // (there's no wire slot to put them on). +// +// Every helper produces a payload rather than writing into the one it was handed, which the +// three frozen-input cases pin: what an overlay is handed at the dial is a record's value. import { test } from 'vitest'; @@ -35,80 +38,80 @@ const msgPayload = (overrides: Partial = {}): Anthropi test('openai-chat-completions: empty rules leave the payload unchanged', () => { const body = ccPayload({ reasoning_effort: 'high', verbosity: 'low', service_tier: 'priority' }); - applyRulesToUpstreamOpenAIChatCompletions(body, {}); - assertEquals(body.reasoning_effort, 'high'); - assertEquals(body.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, {}); + assertEquals(applied.reasoning_effort, 'high'); + assertEquals(applied.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); test('openai-chat-completions: rules stamp every supported native field onto the IR', () => { const body = ccPayload(); - applyRulesToUpstreamOpenAIChatCompletions(body, { + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, { reasoning: { effort: 'high' }, verbosity: 'low', serviceTier: 'priority', }); - assertEquals(body.reasoning_effort, 'high'); - assertEquals(body.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + assertEquals(applied.reasoning_effort, 'high'); + assertEquals(applied.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); test('openai-chat-completions: budget_tokens / adaptive / summary have no native slot — silently dropped', () => { const body = ccPayload(); - applyRulesToUpstreamOpenAIChatCompletions(body, { + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, { reasoning: { budget_tokens: 1024, adaptive: true, summary: 'detailed' }, }); - assertEquals(body.reasoning_effort, undefined); + assertEquals(applied.reasoning_effort, undefined); // Nothing surfaces on the CC IR because none of those rules map to // OpenAI Chat Completions' native fields. - assertEquals('thinking_budget' in body, false); - assertEquals('adaptive_thinking' in body, false); - assertEquals('reasoning_summary' in body, false); + assertEquals('thinking_budget' in applied, false); + assertEquals('adaptive_thinking' in applied, false); + assertEquals('reasoning_summary' in applied, false); }); test('openai-chat-completions: alias rules overwrite existing IR fields', () => { const body = ccPayload({ reasoning_effort: 'low', verbosity: 'high', service_tier: 'default' }); - applyRulesToUpstreamOpenAIChatCompletions(body, { + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, { reasoning: { effort: 'xhigh' }, verbosity: 'low', serviceTier: 'priority', }); - assertEquals(body.reasoning_effort, 'xhigh'); - assertEquals(body.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + assertEquals(applied.reasoning_effort, 'xhigh'); + assertEquals(applied.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); // ── OpenAI Responses target ── test('openai-responses: empty rules leave the payload unchanged', () => { const body = resPayload({ reasoning: { effort: 'high' }, text: { verbosity: 'low' }, service_tier: 'priority' }); - applyRulesToUpstreamOpenAIResponses(body, {}); - assertEquals(body.reasoning?.effort, 'high'); - assertEquals(body.text?.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + const applied = applyRulesToUpstreamOpenAIResponses(body, {}); + assertEquals(applied.reasoning?.effort, 'high'); + assertEquals(applied.text?.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); test('openai-responses: rules stamp every supported native field onto the IR', () => { const body = resPayload(); - applyRulesToUpstreamOpenAIResponses(body, { + const applied = applyRulesToUpstreamOpenAIResponses(body, { reasoning: { effort: 'high', summary: 'concise' }, verbosity: 'medium', serviceTier: 'flex', }); - assertEquals(body.reasoning?.effort, 'high'); - assertEquals(body.reasoning?.summary, 'concise'); - assertEquals(body.text?.verbosity, 'medium'); - assertEquals(body.service_tier, 'flex'); + assertEquals(applied.reasoning?.effort, 'high'); + assertEquals(applied.reasoning?.summary, 'concise'); + assertEquals(applied.text?.verbosity, 'medium'); + assertEquals(applied.service_tier, 'flex'); }); test('openai-responses: budget_tokens / adaptive have no native slot — silently dropped', () => { const body = resPayload(); - applyRulesToUpstreamOpenAIResponses(body, { + const applied = applyRulesToUpstreamOpenAIResponses(body, { reasoning: { budget_tokens: 1024, adaptive: true }, }); - assertEquals(body.reasoning, undefined); - assertEquals('thinking_budget' in body, false); - assertEquals('adaptive_thinking' in body, false); + assertEquals(applied.reasoning, undefined); + assertEquals('thinking_budget' in applied, false); + assertEquals('adaptive_thinking' in applied, false); }); test('openai-responses: alias rules overwrite owned fields while preserving reasoning.context', () => { @@ -117,90 +120,90 @@ test('openai-responses: alias rules overwrite owned fields while preserving reas service_tier: 'default', text: { verbosity: 'high' }, }); - applyRulesToUpstreamOpenAIResponses(body, { + const applied = applyRulesToUpstreamOpenAIResponses(body, { reasoning: { effort: 'xhigh', summary: 'detailed' }, verbosity: 'low', serviceTier: 'priority', }); - assertEquals(body.reasoning, { effort: 'xhigh', summary: 'detailed', context: 'future_mode' }); - assertEquals(body.text?.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + assertEquals(applied.reasoning, { effort: 'xhigh', summary: 'detailed', context: 'future_mode' }); + assertEquals(applied.text?.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); // ── Anthropic Messages target ── test('anthropic-messages: empty rules leave the payload unchanged', () => { const body = msgPayload({ output_config: { effort: 'high' }, thinking: { type: 'enabled', budget_tokens: 512 }, speed: 'fast' }); - applyRulesToUpstreamAnthropicMessages(body, {}); - assertEquals(body.output_config?.effort, 'high'); - assertEquals(body.thinking?.budget_tokens, 512); - assertEquals(body.speed, 'fast'); + const applied = applyRulesToUpstreamAnthropicMessages(body, {}); + assertEquals(applied.output_config?.effort, 'high'); + assertEquals(applied.thinking?.budget_tokens, 512); + assertEquals(applied.speed, 'fast'); }); test('anthropic-messages: effort lands on output_config, budget+adaptive land on thinking', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { effort: 'high', budget_tokens: 2048 }, }); - assertEquals(body.output_config?.effort, 'high'); - assertEquals(body.thinking?.type, 'enabled'); - assertEquals(body.thinking?.budget_tokens, 2048); + assertEquals(applied.output_config?.effort, 'high'); + assertEquals(applied.thinking?.type, 'enabled'); + assertEquals(applied.thinking?.budget_tokens, 2048); }); test('anthropic-messages: verbosity has no Anthropic-shaped slot — silently dropped', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { verbosity: 'low' }); - assertEquals('verbosity' in body, false); + const applied = applyRulesToUpstreamAnthropicMessages(body, { verbosity: 'low' }); + assertEquals('verbosity' in applied, false); }); test('anthropic-messages: summary=concise|detailed collapses onto thinking.display=summarized (enables thinking)', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'concise' } }); - assertEquals(body.thinking?.type, 'enabled'); - assertEquals(body.thinking?.display, 'summarized'); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'concise' } }); + assertEquals(applied.thinking?.type, 'enabled'); + assertEquals(applied.thinking?.display, 'summarized'); }); test('anthropic-messages: summary=omitted collapses onto thinking.display=omitted', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'omitted' } }); - assertEquals(body.thinking?.display, 'omitted'); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'omitted' } }); + assertEquals(applied.thinking?.display, 'omitted'); }); test('anthropic-messages: summary=auto is a no-op (Anthropic default takes over)', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'auto' } }); - assertEquals(body.thinking, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'auto' } }); + assertEquals(applied.thinking, undefined); }); test('anthropic-messages: adaptive=true sets thinking.type=adaptive and ignores budget_tokens', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true, budget_tokens: 4096 } }); - assertEquals(body.thinking?.type, 'adaptive'); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true, budget_tokens: 4096 } }); + assertEquals(applied.thinking?.type, 'adaptive'); }); test('anthropic-messages: adaptive=true strips a client-set budget_tokens from body.thinking', () => { const body = msgPayload(); body.thinking = { type: 'enabled', budget_tokens: 5000 }; - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true } }); - assertEquals(body.thinking?.type, 'adaptive'); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true } }); + assertEquals(applied.thinking?.type, 'adaptive'); // The prior client budget must not leak into the adaptive block — adaptive // auto-determines the budget and a sibling budget_tokens violates the // rules-are-authoritative contract. - assertEquals((body.thinking as { budget_tokens?: number }).budget_tokens, undefined); + assertEquals((applied.thinking as { budget_tokens?: number }).budget_tokens, undefined); }); test('anthropic-messages: serviceTier=fast maps to speed=fast (cross-protocol bridge)', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); - assertEquals(body.speed, 'fast'); - assertEquals(body.service_tier, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); + assertEquals(applied.speed, 'fast'); + assertEquals(applied.service_tier, undefined); }); test('anthropic-messages: non-fast serviceTier lands on service_tier directly', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'priority' }); - assertEquals(body.service_tier, 'priority'); - assertEquals(body.speed, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'priority' }); + assertEquals(applied.service_tier, 'priority'); + assertEquals(applied.speed, undefined); }); test('anthropic-messages: serviceTier=fast clears a pre-existing body.service_tier on the same payload', () => { @@ -209,21 +212,62 @@ test('anthropic-messages: serviceTier=fast clears a pre-existing body.service_ti // semantics for a conflict are undefined. The overlay clears the // sibling field whichever branch it takes. const body = msgPayload({ service_tier: 'priority' }); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); - assertEquals(body.speed, 'fast'); - assertEquals(body.service_tier, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); + assertEquals(applied.speed, 'fast'); + assertEquals(applied.service_tier, undefined); }); test('anthropic-messages: non-fast serviceTier clears a pre-existing body.speed on the same payload', () => { const body = msgPayload({ speed: 'fast' }); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'priority' }); - assertEquals(body.service_tier, 'priority'); - assertEquals(body.speed, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'priority' }); + assertEquals(applied.service_tier, 'priority'); + assertEquals(applied.speed, undefined); }); test('anthropic-messages: alias rules overwrite existing thinking + output_config fields', () => { const body = msgPayload({ output_config: { effort: 'low' }, thinking: { type: 'enabled', budget_tokens: 100 } }); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { effort: 'xhigh', budget_tokens: 9999 } }); - assertEquals(body.output_config?.effort, 'xhigh'); - assertEquals(body.thinking?.budget_tokens, 9999); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { effort: 'xhigh', budget_tokens: 9999 } }); + assertEquals(applied.output_config?.effort, 'xhigh'); + assertEquals(applied.thinking?.budget_tokens, 9999); +}); + +// ── The frozen payload every overlay is actually handed ── +// +// At the dial the payload descends from a record, so the object an overlay is handed is frozen +// and every rule it could want to set is a write that throws. Each of the three is exercised +// with the rules that touch the most of its payload, which is where an in-place write would be. + +test('openai-chat-completions: produces a payload without writing into the frozen one', () => { + const body = Object.freeze(ccPayload({ reasoning_effort: 'low' })); + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, { + reasoning: { effort: 'xhigh' }, + verbosity: 'low', + serviceTier: 'priority', + }); + assertEquals(applied.reasoning_effort, 'xhigh'); + assertEquals(body.reasoning_effort, 'low'); +}); + +test('openai-responses: produces a payload without writing into the frozen one', () => { + const body = Object.freeze(resPayload({ reasoning: { effort: 'low' }, text: { verbosity: 'high' } })); + const applied = applyRulesToUpstreamOpenAIResponses(body, { + reasoning: { effort: 'xhigh', summary: 'detailed' }, + verbosity: 'low', + serviceTier: 'priority', + }); + assertEquals(applied.reasoning?.effort, 'xhigh'); + assertEquals(body.reasoning?.effort, 'low'); +}); + +test('anthropic-messages: produces a payload without writing into the frozen one', () => { + const body = Object.freeze(msgPayload({ thinking: { type: 'enabled', budget_tokens: 100 }, service_tier: 'priority' })); + const applied = applyRulesToUpstreamAnthropicMessages(body, { + reasoning: { effort: 'xhigh', adaptive: true, summary: 'concise' }, + serviceTier: 'fast', + }); + assertEquals(applied.thinking?.type, 'adaptive'); + assertEquals(applied.speed, 'fast'); + assertEquals(applied.service_tier, undefined); + assertEquals(body.thinking?.type, 'enabled'); + assertEquals(body.service_tier, 'priority'); }); diff --git a/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts b/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts index c79128b401..b871cbb332 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts @@ -1,9 +1,9 @@ -// A provider is handed a body it owns. +// A provider is handed the record's own values. // -// Provider interceptors shape their payload in place — Copilot marks individual messages for -// caching — and the record they are built from is deep-frozen. A copy that stopped at the top -// level would satisfy every type in the system and still throw at the dial, so the depth of it -// is the property worth pinning. +// Nothing is deep-copied: the object built here exists to carry the addressed model and then +// drop it, and everything under it is the record's, frozen. A provider that tried to shape its +// payload by writing into what it was given would be writing into a fact — so what is pinned +// here is that the refusal reaches it, at the depth where the rule that used to do it worked. import { describe, it } from 'vitest'; @@ -26,42 +26,47 @@ const recordedRequest = () => { }; describe('bodyForAttempt', () => { - it('hands over children a provider can write into', () => { - const body = bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), () => {}); - - // What the Copilot cache-control rule does, at the node it does it to. - const message = body.messages[0] as Record; - message.copilot_cache_control = { type: 'ephemeral' }; - assertEquals(message.copilot_cache_control, { type: 'ephemeral' }); - }); - - it('leaves the record it was built from untouched', () => { + it('hands over children no one can write into', () => { const recorded = recordedRequest(); - const body = bodyForAttempt(recorded, candidateFor('upstream-model'), () => {}); - (body.messages[0] as Record).copilot_cache_control = { type: 'ephemeral' }; + const body = bodyForAttempt(recorded, candidateFor('upstream-model'), body => body); + + // What the Copilot cache-control rule used to do, at the node it did it to. It rebuilds + // now, and this is why: the message it was reaching into is the record's own. + let refused: unknown; + try { + (body.messages[0] as Record).copilot_cache_control = { type: 'ephemeral' }; + } catch (error) { refused = error; } + assertEquals(refused instanceof TypeError, true); assertEquals('copilot_cache_control' in recorded.messages[0], false); assertEquals(recorded.model, 'alias-name'); }); it('addresses the model the candidate resolved and drops the one the client named', () => { - const body = bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), () => {}); + const body = bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), body => body); assertEquals('model' in body, false); }); - it('applies the alias rules over the copy', () => { + it('applies the alias rules over the body it addressed', () => { const rules = { reasoning: { effort: 'high' } } as NonNullable; + let addressed: string | undefined; const body = bodyForAttempt( recordedRequest(), candidateFor('upstream-model', rules), - (payload, applied) => { (payload as Record).reasoning_effort = applied.reasoning?.effort; }, + (payload, applied) => { + addressed = payload.model; + return { ...payload, reasoning_effort: applied.reasoning?.effort }; + }, ); assertEquals((body as Record).reasoning_effort, 'high'); + // The overlay ran over the model this attempt is for rather than the alias the client + // addressed, which is what lets a rule speak about the upstream it is actually dialling. + assertEquals(addressed, 'upstream-model'); }); it('applies nothing where the alias carries no rules', () => { let ran = false; - bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), () => { ran = true; }); + bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), body => { ran = true; return body; }); assertEquals(ran, false); }); }); diff --git a/packages/gateway/src/data-plane/chat/shared/alias-rules.ts b/packages/gateway/src/data-plane/chat/shared/alias-rules.ts index 8f75b71c87..ac98969b6f 100644 --- a/packages/gateway/src/data-plane/chat/shared/alias-rules.ts +++ b/packages/gateway/src/data-plane/chat/shared/alias-rules.ts @@ -1,95 +1,104 @@ // Post-translate rule overlay. The alias resolver tags each alias-origin // candidate with `.rules`; each terminal wire call reads them off the -// dispatching candidate and writes onto the target IR's NATIVE slot before +// dispatching candidate and puts them on the target IR's NATIVE slot before // dispatching. Rules that a target protocol cannot express are silently // dropped — the wire has nowhere to put them. // // Structuring the overlay this way keeps every translate pair pure // native↔native and eliminates the fan-out of Floway-extension fields onto // each source IR. +// +// Each overlay returns the payload it produced rather than writing into the one it was handed. +// What it is handed descends from a record, and a record's values are frozen — an overlay that +// wrote in place would work only for as long as its caller happened to have rebuilt the level +// it writes to, which is the shape of failure that reached a client as a 502 once already. import type { AnthropicMessagesPayload, AnthropicMessagesThinkingDisplay } from '@floway-dev/protocols/anthropic-messages'; import type { AliasRules } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -const hasReasoning = (rules: AliasRules): rules is AliasRules & { reasoning: NonNullable } => - rules.reasoning !== undefined; +type Reasoning = NonNullable; -export const applyRulesToUpstreamOpenAIChatCompletions = (body: OpenAIChatCompletionsPayload, rules: AliasRules): void => { - if (hasReasoning(rules)) { - const { effort } = rules.reasoning; - if (effort !== undefined) body.reasoning_effort = effort; - // `budget_tokens`, `adaptive`, and `summary` have no native OpenAI Chat Completions - // Completions slot; drop silently. - } - if (rules.verbosity !== undefined) body.verbosity = rules.verbosity; - if (rules.serviceTier !== undefined) body.service_tier = rules.serviceTier; +const reasoningOf = (rules: AliasRules): Reasoning => rules.reasoning ?? {}; + +export const applyRulesToUpstreamOpenAIChatCompletions = (body: T, rules: AliasRules): T => { + // `budget_tokens`, `adaptive`, and `summary` have no native OpenAI Chat Completions slot; + // drop silently. + const { effort } = reasoningOf(rules); + return { + ...body, + ...(effort === undefined ? {} : { reasoning_effort: effort }), + ...(rules.verbosity === undefined ? {} : { verbosity: rules.verbosity }), + ...(rules.serviceTier === undefined ? {} : { service_tier: rules.serviceTier }), + } as T; }; -export const applyRulesToUpstreamOpenAIResponses = (body: OpenAIResponsesPayload, rules: AliasRules): void => { - if (hasReasoning(rules)) { - const { effort, summary } = rules.reasoning; - if (effort !== undefined || summary !== undefined) { - const existing = body.reasoning ?? {}; - body.reasoning = { - ...existing, - ...(effort !== undefined ? { effort } : {}), - ...(summary !== undefined ? { summary } : {}), - }; - } - // `budget_tokens` and `adaptive` have no native OpenAI Responses slot; drop - // silently. - } - if (rules.verbosity !== undefined) { - body.text = { ...body.text, verbosity: rules.verbosity }; - } - if (rules.serviceTier !== undefined) body.service_tier = rules.serviceTier; +export const applyRulesToUpstreamOpenAIResponses = (body: T, rules: AliasRules): T => { + // `budget_tokens` and `adaptive` have no native OpenAI Responses slot; drop silently. + const { effort, summary } = reasoningOf(rules); + return { + ...body, + ...(effort === undefined && summary === undefined ? {} : { + reasoning: { + ...body.reasoning, + ...(effort === undefined ? {} : { effort }), + ...(summary === undefined ? {} : { summary }), + }, + }), + ...(rules.verbosity === undefined ? {} : { text: { ...body.text, verbosity: rules.verbosity } }), + ...(rules.serviceTier === undefined ? {} : { service_tier: rules.serviceTier }), + } as T; }; -export const applyRulesToUpstreamAnthropicMessages = (body: AnthropicMessagesPayload, rules: AliasRules): void => { - if (hasReasoning(rules)) { - const { effort, budget_tokens, adaptive, summary } = rules.reasoning; +export const applyRulesToUpstreamAnthropicMessages = (body: T, rules: AliasRules): T => { + // `verbosity` has no native Anthropic Messages slot; drop silently. + const reasoning = reasoningOf(rules); + const thinking = anthropicMessagesThinking(body.thinking, reasoning); + const reasoned = { + ...body, // Anthropic stores explicit effort in `output_config.effort`; budget / // adaptive ride on `thinking.*`. Splitting them so both can be set in // the same overlay (effort fixed + budget pinned, e.g.) without one // erasing the other. - if (effort !== undefined) { - body.output_config = { ...body.output_config, effort }; - } - const display = summary !== undefined ? mapSummaryToAnthropicMessagesDisplay(summary) : undefined; - const displayPart = display !== undefined ? { display } : {}; - if (adaptive === true) { - // Adaptive auto-determines the budget; strip any client-set - // `budget_tokens` so the alias rule's mode isn't accompanied by a - // sibling budget the operator didn't ask for. - const { budget_tokens: _drop, ...priorThinking } = body.thinking ?? {}; - body.thinking = { ...priorThinking, type: 'adaptive', ...displayPart }; - } else if (budget_tokens !== undefined) { - body.thinking = { ...body.thinking, type: 'enabled', budget_tokens, ...displayPart }; - } else if (display !== undefined) { - // Anthropic discards `thinking.display` unless a mode is set; default - // to the enabled variant so the summary hint reaches the wire. - body.thinking = { ...body.thinking, type: 'enabled', ...displayPart }; - } - } - // `verbosity` has no native Anthropic Messages slot; drop silently. - if (rules.serviceTier !== undefined) { - // The cross-protocol bridge in translate maps `speed: 'fast'` ↔ - // `service_tier: 'fast'`; on a native Anthropic Messages target the alias rule - // `serviceTier: 'fast'` lands on `speed` so the upstream sees Fast Mode - // through its native field. Other tier values pass through on - // `service_tier` since Anthropic Messages's native enum (`auto`/`standard_only`) - // doesn't model them. Whichever branch we take, clear the sibling field - // so the upstream never sees two tiers in conflict. - if (rules.serviceTier === 'fast') { - body.speed = 'fast'; - delete body.service_tier; - } else { - body.service_tier = rules.serviceTier; - delete body.speed; - } + ...(reasoning.effort === undefined ? {} : { output_config: { ...body.output_config, effort: reasoning.effort } }), + ...(thinking === undefined ? {} : { thinking }), + } as T; + if (rules.serviceTier === undefined) return reasoned; + + // The cross-protocol bridge in translate maps `speed: 'fast'` ↔ + // `service_tier: 'fast'`; on a native Anthropic Messages target the alias rule + // `serviceTier: 'fast'` lands on `speed` so the upstream sees Fast Mode + // through its native field. Other tier values pass through on + // `service_tier` since Anthropic Messages's native enum (`auto`/`standard_only`) + // doesn't model them. Whichever branch we take, the sibling field is gone + // so the upstream never sees two tiers in conflict. + const { speed: _speed, service_tier: _serviceTier, ...tierless } = reasoned; + return (rules.serviceTier === 'fast' + ? { ...tierless, speed: 'fast' } + : { ...tierless, service_tier: rules.serviceTier }) as T; +}; + +/** The `thinking` block an overlay asks for, or `undefined` where it asks for none. */ +const anthropicMessagesThinking = ( + prior: AnthropicMessagesPayload['thinking'], + reasoning: Reasoning, +): AnthropicMessagesPayload['thinking'] | undefined => { + const { budget_tokens, adaptive, summary } = reasoning; + const display = summary === undefined ? undefined : mapSummaryToAnthropicMessagesDisplay(summary); + const displayPart = display === undefined ? {} : { display }; + + if (adaptive === true) { + // Adaptive auto-determines the budget; a client-set `budget_tokens` goes, so the alias + // rule's mode is not accompanied by a sibling budget the operator didn't ask for. + const { budget_tokens: _drop, ...priorThinking } = prior ?? {}; + return { ...priorThinking, type: 'adaptive', ...displayPart }; } + if (budget_tokens !== undefined) return { ...prior, type: 'enabled', budget_tokens, ...displayPart }; + // Anthropic discards `thinking.display` unless a mode is set; default + // to the enabled variant so the summary hint reaches the wire. + if (display !== undefined) return { ...prior, type: 'enabled', ...displayPart }; + return undefined; }; // Collapse OpenAI-style summary presets onto Anthropic's structured diff --git a/packages/gateway/src/data-plane/pipeline/attempt-body.ts b/packages/gateway/src/data-plane/pipeline/attempt-body.ts index 47633a4008..e84c7a300c 100644 --- a/packages/gateway/src/data-plane/pipeline/attempt-body.ts +++ b/packages/gateway/src/data-plane/pipeline/attempt-body.ts @@ -1,25 +1,27 @@ // What a provider is handed when a stage dials it. // -// The two sides of this boundary disagree about ownership, and both are right. A record's -// values are frozen — a fact that could be edited after the event is not a fact — while a -// provider's interceptors shape the body they are given in place, down to nested nodes: -// Copilot writes `copilot_cache_control` onto individual messages, and its initiator rule -// writes into a nested field of an Anthropic Messages payload. +// The record's own children travel, by identity. One object is built here, and only because +// this is where the addressed model is stamped and then dropped; everything under it is the +// record's, frozen — a fact that could be edited after the event is not a fact — and a provider +// shapes the body it sends by rebuilding rather than by writing into what it was given. So both +// sides of this boundary agree and there is nothing to defend against. // -// Spreading a frozen fact produces a fresh top level over frozen children, so those writes -// throw — as a 502 carrying `Cannot add property …, object is not extensible`, raised at the -// point in the stack least able to explain it. What crosses this boundary is therefore a copy -// the provider owns outright, children included. +// It was not always so, and the shape of the failure is worth keeping: three Copilot rules +// wrote one level down, into a message or a content block, which against a frozen record threw +// `Cannot add property …, object is not extensible` — as a 502, from the point in the stack +// least able to explain it. A deep copy here absorbed that, at the cost of one pass over every +// payload on every dial, and at the larger cost of making immutability something the gateway +// worked around. The three rules rebuild now, and so does every alias overlay: an overlay that +// wrote in place would be safe only for as long as this function happened to have rebuilt the +// level it writes to, which is the same trap one level up. // -// The copy costs one pass over the payload, beside the serialization the dial performs -// immediately afterwards. What it buys is that a provider may keep being written the way every -// provider here is already written. +// So what is left is only the three things every wire was otherwise writing by hand. import type { ModelCandidate } from '@floway-dev/provider'; /** - * The body for this attempt: the record's request, copied for the provider, addressed to the - * model the candidate resolved, with the alias' own rules applied over it. + * The body for this attempt: the record's request, addressed to the model the candidate + * resolved, with the alias' own rules applied over it. * * The id the client addressed does not travel. An alias is a gateway concept — the provider * re-stamps whatever it resolved upstream — so the key is dropped rather than forwarded, and @@ -28,10 +30,10 @@ import type { ModelCandidate } from '@floway-dev/provider'; export const bodyForAttempt = ( recorded: T, candidate: ModelCandidate, - applyRules: (body: T, rules: NonNullable) => void, + applyRules: (body: T, rules: NonNullable) => T, ): Omit => { - const payload = { ...structuredClone(recorded), model: candidate.model.id } as T; - if (candidate.rules !== undefined) applyRules(payload, candidate.rules); + const addressed = { ...recorded, model: candidate.model.id } as T; + const payload = candidate.rules === undefined ? addressed : applyRules(addressed, candidate.rules); const { model: _addressed, ...body } = payload; return body; }; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts index f7459acf79..cc4dbb9624 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts @@ -1,12 +1,13 @@ import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; import type { AnthropicMessagesAssistantInputContentBlock, + AnthropicMessagesPayload, AnthropicMessagesTextBlock, AnthropicMessagesUserContentBlock, } from '@floway-dev/protocols/anthropic-messages'; /** - * Anthropic's Anthropic Messages API defines a top-level `cache_control` field that + * Anthropic's Messages API defines a top-level `cache_control` field that * "automatically applies a cache_control marker to the last cacheable block * in the request" (per `MessageCreateParamsBase` in the official SDK). * Copilot's `/v1/messages` deployment validates against an older schema for @@ -35,29 +36,51 @@ type CacheableBlock = Extract< const isCacheableBlock = (block: AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock): block is CacheableBlock => block.type === 'text' || block.type === 'image' || block.type === 'tool_use' || block.type === 'tool_result'; +/** Where the marker lands: the last message that can carry one, and within it either the whole + * string content or the last cacheable block. Reading the position out first is what lets the + * rewrite below rebuild one path and leave every other message alone. */ +const markerSite = (messages: AnthropicMessagesPayload['messages']): { readonly message: number; readonly block: number | 'whole' } | undefined => { + for (let m = messages.length - 1; m >= 0; m--) { + const content = messages[m].content; + if (typeof content === 'string') return { message: m, block: 'whole' }; + for (let b = content.length - 1; b >= 0; b--) { + if (isCacheableBlock(content[b])) return { message: m, block: b }; + } + } + return undefined; +}; + export const withTopLevelCacheControlApplied: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { const payload = ctx.payload as typeof ctx.payload & { cache_control?: { type: 'ephemeral' } }; const topLevel = payload.cache_control; if (topLevel === undefined) return await run(); - delete payload.cache_control; - - for (let m = payload.messages.length - 1; m >= 0; m--) { - const message = payload.messages[m]; - - if (typeof message.content === 'string') { - const block: AnthropicMessagesTextBlock = { type: 'text', text: message.content, cache_control: topLevel }; - message.content = [block] as AnthropicMessagesUserContentBlock[] | AnthropicMessagesAssistantInputContentBlock[]; - return await run(); - } - - for (let b = message.content.length - 1; b >= 0; b--) { - const block = message.content[b]; - if (!isCacheableBlock(block)) continue; - block.cache_control ??= topLevel; - return await run(); - } + const { cache_control: _ported, ...withoutTopLevel } = payload; + const site = markerSite(payload.messages); + if (site === undefined) { + ctx.payload = withoutTopLevel; + return await run(); } + ctx.payload = { + ...withoutTopLevel, + // Each rewrite keeps its message's own role and only re-homes that role's own blocks, so + // the array is handed back at its element type; spreading alone widens it to the union. + messages: payload.messages.map((message, m): AnthropicMessagesPayload['messages'][number] => { + if (m !== site.message) return message; + if (site.block === 'whole') { + const block: AnthropicMessagesTextBlock = { type: 'text', text: message.content as string, cache_control: topLevel }; + return { ...message, content: [block] } as AnthropicMessagesPayload['messages'][number]; + } + const content = message.content as readonly CacheableBlock[]; + // `??=` semantics: a block that already states its own marker keeps it. + if (content[site.block].cache_control !== undefined) return message; + return { + ...message, + content: content.map((block, b) => (b === site.block ? { ...block, cache_control: topLevel } : block)), + } as AnthropicMessagesPayload['messages'][number]; + }), + }; + return await run(); }; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts index 99ac929c50..69901a4bdc 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts @@ -1,4 +1,5 @@ import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; +import { mapKeepingIdentity, withKeysChanged } from '../shared/rebuild.ts'; /** * Two `cache_control` sub-fields are beta extensions to the base @@ -32,35 +33,36 @@ import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; * - https://github.com/caozhiyuan/copilot-api/issues/269 * - https://github.com/caozhiyuan/copilot-api/commit/ce8224c55933f811abe5bf9ba42f9336a7852997 */ -const stripExtensions = (block: Record): void => { - const cacheControl = block.cache_control; - if (!cacheControl || typeof cacheControl !== 'object') return; +/** The block with its cache-control extensions gone, or the same block when it carried none. + * Returning the original is what lets an untouched message ride through by identity. */ +const stripExtensions = (block: T): T => { + const cacheControl = (block as Record).cache_control; + if (!cacheControl || typeof cacheControl !== 'object') return block; const { scope: _scope, ttl: _ttl, ...rest } = cacheControl as Record; - if (Object.keys(rest).length > 0) block.cache_control = rest; - else delete block.cache_control; + return withKeysChanged(block, { cache_control: Object.keys(rest).length > 0 ? rest : undefined }); }; export const withCacheControlExtensionsStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - if (Array.isArray(ctx.payload.system)) { - for (const block of ctx.payload.system as unknown as Record[]) { - stripExtensions(block); - } - } + const payload = ctx.payload; - if (ctx.payload.tools) { - for (const tool of ctx.payload.tools as unknown as Record[]) { - stripExtensions(tool); - } - } + const system = Array.isArray(payload.system) + ? mapKeepingIdentity(payload.system, stripExtensions) + : payload.system; + const tools = payload.tools === undefined ? payload.tools : mapKeepingIdentity(payload.tools, stripExtensions); + const messages = mapKeepingIdentity(payload.messages, message => ( + Array.isArray(message.content) + // The content arrays are a union of per-role block types, so the walk is expressed over + // the shape they share — an object — and handed back at the array's own type. + ? withKeysChanged(message, { content: mapKeepingIdentity(message.content as readonly object[], stripExtensions) }) + : message + )); - for (const message of ctx.payload.messages) { - if (!Array.isArray(message.content)) continue; - - for (const block of message.content as unknown as Record[]) { - stripExtensions(block); - } - } + ctx.payload = withKeysChanged(payload, { + ...(system === payload.system ? {} : { system }), + ...(tools === payload.tools ? {} : { tools }), + ...(messages === payload.messages ? {} : { messages }), + }); return await run(); }; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts index 0c67d7c9ce..923e1ea4d2 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts +++ b/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts @@ -64,12 +64,19 @@ const selectCacheMarkerIndexes = (messages: readonly OpenAIChatCompletionsMessag }; export const withCacheControlMarkersAttached: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { - const indexes = selectCacheMarkerIndexes(ctx.payload.messages); - for (const index of indexes) { - // Fresh object per message so downstream mutations (none today, but - // cheap insurance) cannot bleed across messages. - (ctx.payload.messages[index] as CopilotCacheableMessage).copilot_cache_control = { ...COPILOT_CONTEXT_CACHE_CONTROL }; - } + const marked = new Set(selectCacheMarkerIndexes(ctx.payload.messages)); + if (marked.size === 0) return await run(); + + ctx.payload = { + ...ctx.payload, + // A marker is its own object per message, so nothing downstream can carry a change from + // one to another. The messages that were not selected ride through by identity. + messages: ctx.payload.messages.map((message, index) => ( + marked.has(index) + ? { ...message, copilot_cache_control: { ...COPILOT_CONTEXT_CACHE_CONTROL } } as CopilotCacheableMessage + : message + )), + }; return await run(); }; diff --git a/packages/provider-copilot/src/interceptors/shared/rebuild.ts b/packages/provider-copilot/src/interceptors/shared/rebuild.ts new file mode 100644 index 0000000000..db6d4e11a7 --- /dev/null +++ b/packages/provider-copilot/src/interceptors/shared/rebuild.ts @@ -0,0 +1,38 @@ +// Rebuilding a payload instead of writing into one. +// +// A boundary interceptor is handed the record's own value, and the record is frozen — a fact +// that could be edited after the event is not a fact. So a rule that shapes the outgoing +// payload returns a new one rather than reaching into the old, and every provider here +// already does that at the top level: `ctx.payload = { ...ctx.payload, … }`. +// +// What these two add is the same discipline one level down, without paying for a copy of the +// whole tree. A rewrite that changed nothing hands the same object back, so an untouched +// message or block rides through by identity and only the path that actually changed is +// rebuilt — which is what keeps a long conversation costing a handful of objects rather than +// a full clone, and is the same rule the record itself is folded by. + +/** Applies `rewrite` to every item, and hands back the original array when none of them + * changed — so "nothing happened here" is the same test at every level. */ +export const mapKeepingIdentity = (items: readonly T[], rewrite: (item: T) => T): readonly T[] => { + let changed = false; + const next = items.map(item => { + const rewritten = rewrite(item); + if (rewritten !== item) changed = true; + return rewritten; + }); + return changed ? next : items; +}; + +/** The object form: rebuild with `patch` applied, or hand back the original when the patch is + * empty. `undefined` in the patch removes the key, which is what a rest-destructuring would + * do and what `delete` used to do in place. */ +export const withKeysChanged = (source: T, patch: Readonly>): T => { + const keys = Object.keys(patch); + if (keys.length === 0) return source; + const next: Record = { ...(source as Record) }; + for (const key of keys) { + if (patch[key] === undefined) delete next[key]; + else next[key] = patch[key]; + } + return next as T; +}; From d26fb1f0cef2fb64e26fbb2d513027a80b29fc98 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 07:14:49 +0800 Subject: [PATCH 087/156] refactor(chat): meter a wire's answer above its rules, not at its dial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A wire's rules rewrite the answer on the way up: a usage block gets moved onto the carrier chunk the spec puts it on, cache buckets an upstream reports alongside the input count get folded back into it, and a vendor's own spelling of both becomes OpenAI's. What the client reads is the result of all of that. The row was written from the frames before any of it. The meter lived inside the dial, at the bottom of the wire, so it read what the upstream sent rather than what the wire settled — and the two disagree exactly where a normalizer had something to say. A DeepSeek-shaped upstream is the worked case: it counts its cached prefix outside the input total and names it in its own fields, so a turn the client was shown as 13791 input with 13312 cached was billed as 479 input with no cache read at all. The prefix was not mispriced; it was not billed. So the meter is a stage of its own now, at the head of each wire — outermost, so the frames reach it having been through every rewrite. There is no seam below it left where a figure could be taken early, because the dial no longer names the streamed-usage key at all: a stage cannot report a figure it has no means to produce, and that is what keeps the class from returning the next time a rule is added. `normalize-exclusive-cached-tokens` is one such rule, and it now runs below the meter by construction rather than by anyone remembering to order them. The stage shape is shared and each protocol contributes only the reading of its own events, which is the one thing that differs. Which identity the figure is attributed to comes from the entity the dial recorded when it called an upstream — already the statement "this attempt bills to this identity, nothing reported yet" — so the meter fills in what it turned out to be billable for rather than deriving a second identity of its own. A wire that hands up a stream was dialled, so there is exactly one; anything else throws rather than billing to a guess. Two answers are not streams and neither is metered: a refusal, and the single envelope a compaction is. Both state their own counts or have none, so the reading is `null` and settlement writes the row now instead of waiting. On the Anthropic Messages wire nothing observable changes — it has no response-side rule today. The placement is what makes that stay true. Verified by mutation on both wires that have one: moving the meter back to the dial's own position fails the new OpenAI Chat Completions row with the cache read missing entirely, and the new OpenAI Responses row the same way. --- .../openai-chat-completions-pipeline_test.ts | 10 ++ .../chat/openai-responses-pipeline_test.ts | 33 ++++++ .../chat/anthropic-messages/pipeline.ts | 28 +++-- packages/gateway/src/data-plane/chat/meter.ts | 106 ++++++++++++++++++ .../chat/openai-chat-completions/pipeline.ts | 29 ++--- .../chat/openai-responses/pipeline.ts | 24 ++-- 6 files changed, 197 insertions(+), 33 deletions(-) create mode 100644 packages/gateway/src/data-plane/chat/meter.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts index e863b53eff..1b3fd9273a 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts @@ -344,6 +344,16 @@ describe('the chat completions chain', () => { total_tokens: 14164, prompt_tokens_details: { cached_tokens: 13312 }, }); + // And the row is written from that same frame rather than from the vendor's own. The + // reading is taken above every rule, so what is billed and what is shown cannot disagree — + // metering at the dial would bill the 479 DeepSeek called the input and lose the cached + // prefix entirely, against the 13791 the client was just shown. + const outcome = await facts['response.chat.openaiChatCompletions.streamedUsage']!; + expect(outcome.billable[0]!.quantities).toEqual({ + input_tokens: '479', + input_cache_read_tokens: '13312', + output_tokens: '373', + }); }); // The other two dialects the wire composes, each stating it in a different direction: Qwen diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts index 622bc00dbe..aee45a84e2 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -398,6 +398,39 @@ describe('the responses chain', () => { expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 11 }); }); + // The reading is taken above the wire's rules, so what the row is written from is the same + // frame the client was shown. An upstream counting its cached prefix outside the input total + // is where the two would otherwise come apart: 479 + 13312 + 373 is the 14164 it states, so + // the fold restores OpenAI's inclusive contract before anything reads the numbers. + it('bills the usage its rules settled rather than the one the upstream sent', async () => { + resolves([candidate({ + callOpenAIResponses: async () => stream( + delta('hi'), + completed('hi', { + input_tokens: 479, output_tokens: 373, total_tokens: 14164, + input_tokens_details: { cached_tokens: 13312 }, + } as OpenAIResponsesResult['usage']), + ), + })]); + + const { facts } = await serve(true); + const frames = withoutCarrier(await drain(facts['response.chat.openaiResponses.rendered'])); + const terminal = frames + .filter(frame => frame.event === 'response.completed') + .map(frame => JSON.parse(frame.data) as { response: OpenAIResponsesResult }); + + // What the client was shown: the cached prefix folded back into the input total. + expect(terminal[0]!.response.usage).toMatchObject({ input_tokens: 13791, output_tokens: 373 }); + // And what the row is written from, which is that same figure split onto the two metrics a + // cache read is priced on. Metering at the dial would bill 479 and lose the prefix. + const outcome = await facts['response.chat.openaiResponses.streamedUsage']!; + expect(outcome.billable[0]!.quantities).toEqual({ + input_tokens: '479', + input_cache_read_tokens: '13312', + output_tokens: '373', + }); + }); + // Time to first token is measured where the token is — a lifecycle envelope is not one, and // a run that never stamps it is recorded as having produced nothing to time. it('stamps the first frame that carried a generated token', async () => { diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 24876e673d..a0bccfed31 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -43,6 +43,7 @@ import { disableReasoningOnForcedToolChoiceForAnthropicMessages, stripBillingAttributionFromAnthropicMessages, } from '../interceptors.ts'; +import { meterChatWire } from '../meter.ts'; import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; @@ -252,16 +253,20 @@ const answerClaudeCodeProbe = defineStage< }, }); -const callAnthropicMessagesUpstream = (streamedUsage: string) => defineStage< +/** + * The wire. What it hands up is the upstream's own frames, unread: the reading is taken at the + * top of the wire, above every rule that rewrites them, so nothing here has the means to + * produce a figure the client will not be shown. + */ +const callAnthropicMessagesUpstream = defineStage< M<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, - M<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers'> & Record, + M<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'callAnthropicMessagesUpstream', return: { provides: [ 'response.chat.anthropicMessages', - streamedUsage, 'response.usage.billable', 'response.http.headers', ], @@ -306,7 +311,6 @@ const callAnthropicMessagesUpstream = (streamedUsage: string) => defineStage< return move({ ...facts, 'response.chat.anthropicMessages': { status: 502, message: error instanceof Error ? error.message : String(error) }, - [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], }); @@ -329,7 +333,6 @@ const callAnthropicMessagesUpstream = (streamedUsage: string) => defineStage< message: text, ...(parsed === undefined ? {} : { body: parsed }), }, - [streamedUsage]: null, 'response.usage.billable': called, 'response.http.headers': [...result.response.headers], }); @@ -338,11 +341,9 @@ const callAnthropicMessagesUpstream = (streamedUsage: string) => defineStage< // This candidate answered, so it is the one a follow-up turn carrying our own state // must come back to. use.selectAffinity(candidate); - const metered = meterAnthropicMessages(result.events, identity, use.gateway.attempt); return move({ ...facts, - 'response.chat.anthropicMessages': { kind: 'stream' as const, frames: metered.frames }, - [streamedUsage]: metered.outcome, + 'response.chat.anthropicMessages': { kind: 'stream' as const, frames: result.events }, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); @@ -357,12 +358,21 @@ const callAnthropicMessagesUpstream = (streamedUsage: string) => defineStage< * speaks about *this* wire belong here. The system-role rewrite states what an upstream's Anthropic Messages * endpoint accepts, so it applies to whatever body this wire actually sends and to nothing * that leaves for another protocol. + * + * The meter is above every one of them, which is what makes the figure it bills the one the + * client is shown. */ export const anthropicMessagesWire = (streamedUsage: string): readonly Stage[] => [ + meterChatWire({ + wire: 'anthropicMessages', + answer: 'response.chat.anthropicMessages', + streamedUsage, + read: meterAnthropicMessages, + }), stripBillingAttributionFromAnthropicMessages, disableReasoningOnForcedToolChoiceForAnthropicMessages, applyRoleCompatibilityToAnthropicMessages, - callAnthropicMessagesUpstream(streamedUsage), + callAnthropicMessagesUpstream, ]; /** This family's own reading, which every wire under it hands up. */ diff --git a/packages/gateway/src/data-plane/chat/meter.ts b/packages/gateway/src/data-plane/chat/meter.ts new file mode 100644 index 0000000000..35b1092db0 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/meter.ts @@ -0,0 +1,106 @@ +// The meter, as a wire's outermost stage. +// +// A wire's rules rewrite the answer on the way up: a usage block gets moved onto the carrier +// chunk the spec puts it on, cache buckets an upstream reports alongside the input count get +// folded back into it, and a vendor's own spelling of both becomes OpenAI's. What the client +// reads is the result of all of that. +// +// So that is what the row must be written from. Reading the upstream's raw frames instead +// bills one number and shows another for the same turn, and the two disagree exactly where a +// normalizer had something to say — which is every upstream that needs one. Placing the meter +// **above** the rules rather than at the dial is what makes the two readings the same reading: +// the frames reach it having been through every rewrite, so there is no seam below it left +// where a figure could be taken early. +// +// It is also why the dial no longer names the streamed-usage key at all. A stage cannot report +// a figure it has no means to produce, and stopping the dial from producing one is what keeps +// the class from coming back the next time a rule is added. + +import type { ChatAnswer, ChatFacts } from './facts.ts'; +import type { ChatServices } from './stages.ts'; +import type { BillableEntity } from '../pipeline/facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import type { StreamOutcome } from '../pipeline/serve.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +type ResponseKey = Extract; + +/** What one protocol contributes: reading its own events. Everything else about metering — + * where the reading is taken, which identity it is attributed to, what a run that never + * streamed reports — is the same for all of them and lives here. */ +export type ChatMeterReading = ( + frames: AsyncIterable>, + identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, +) => { readonly frames: AsyncIterable>; readonly outcome: Promise }; + +/** Which identity the figure is attributed to, read off the entity the dial recorded when it + * called an upstream. That entity is already the statement "this attempt bills to this + * identity, and nothing has been reported for it yet" — so the meter fills in what it turned + * out to be billable for rather than deriving a second identity of its own. + * + * A wire that hands up a stream has been dialled, so there is exactly one. Anything else is + * this gateway contradicting itself, and it says so rather than billing to a guess. */ +const attributedTo = (billable: readonly BillableEntity[], wire: string): TelemetryModelIdentity => { + if (billable.length !== 1) { + throw new Error(`${wire} handed up a stream billable to ${billable.length} entities, and a metered stream bills to the one that was dialled`); + } + return billable[0].identity; +}; + +/** + * Meters whatever the wire below hands up. + * + * A refusal and a collected body have no frames to read, and both are ordinary: the reading is + * `null`, which is how a family says settlement has nothing left to wait for and should write + * the row now. + */ +export const meterChatWire = (spec: { + /** The wire this meters, which names the stage — so a dump says which protocol's events the + * reading was taken off without the name being written twice. */ + readonly wire: string; + /** The answer this wire hands up, whose frames carry the upstream's own usage. */ + readonly answer: ResponseKey; + /** Where the reading lands. It is the *source* family's key rather than this protocol's, + * which is why the wire is told it rather than naming one. */ + readonly streamedUsage: string; + readonly read: ChatMeterReading; +}) => defineStage< + Record, + Record, + { [K in ResponseKey]?: ChatAnswer } & { 'response.usage.billable': readonly BillableEntity[] } & Record, + { [K in ResponseKey]?: ChatAnswer } & Record, + ChatServices +>({ + name: `meter:${spec.wire}`, + through: { + // Nothing on the way down. A meter is a reader of what came back, and naming a request key + // here would put it in the entry contract of every chain that dials this wire. + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: [spec.answer, 'response.usage.billable'], + consumes: [], + provides: [spec.answer, spec.streamedUsage], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const answer = back[spec.answer] as ChatAnswer; + if (isFailure(answer) || answer.kind !== 'stream') { + return move({ ...back, [spec.streamedUsage]: null }) as never; + } + + const metered = spec.read( + answer.frames as AsyncIterable>, + attributedTo(back['response.usage.billable'], spec.wire), + use.gateway.attempt, + ); + return move({ + ...back, + [spec.answer]: { kind: 'stream' as const, frames: metered.frames }, + [spec.streamedUsage]: metered.outcome, + }) as never; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index 33432b0982..1e0d9edc43 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -48,6 +48,7 @@ import { vendorKimiNormalizeForOpenAIChatCompletions, vendorQwenNormalizeForOpenAIChatCompletions, } from '../interceptors.ts'; +import { meterChatWire } from '../meter.ts'; import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamOpenAIChatCompletions } from '../shared/alias-rules.ts'; @@ -193,20 +194,19 @@ const renderSSE = ( * chain: both hand up `response.chat.openaiChatCompletions`, and the stage above cannot tell which * ran. * - * `streamedUsage` is the one key it is told. A wire hands up one reading whatever protocol it - * spoke, and where that reading lands is the *source* family's own key rather than this - * protocol's — so it is built with it rather than naming one of its own. + * What it hands up is the upstream's own frames, unread. The reading is taken at the top of the + * wire, above every rule that rewrites them, so nothing here has the means to produce a figure + * the client will not be shown. */ -const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage< +const callOpenAIChatCompletionsUpstream = defineStage< C<'request.chat.openaiChatCompletions' | 'route.attempt' | 'ingress.http.headers'>, - C<'response.chat.openaiChatCompletions' | 'response.usage.billable' | 'response.http.headers'> & Record, + C<'response.chat.openaiChatCompletions' | 'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'callOpenAIChatCompletionsUpstream', return: { provides: [ 'response.chat.openaiChatCompletions', - streamedUsage, 'response.usage.billable', 'response.http.headers', ], @@ -240,7 +240,6 @@ const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage return move({ ...facts, 'response.chat.openaiChatCompletions': { status: 502, message: error instanceof Error ? error.message : String(error) }, - [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], }); @@ -263,7 +262,6 @@ const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage message: text, ...(parsed === undefined ? {} : { body: parsed }), }, - [streamedUsage]: null, 'response.usage.billable': called, 'response.http.headers': [...result.response.headers], }); @@ -272,11 +270,9 @@ const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage // This candidate answered, so it is the one a follow-up turn carrying our own state // must come back to. use.selectAffinity(candidate); - const metered = meterOpenAIChatCompletions(result.events, identity, use.gateway.attempt); return move({ ...facts, - 'response.chat.openaiChatCompletions': { kind: 'stream' as const, frames: metered.frames }, - [streamedUsage]: metered.outcome, + 'response.chat.openaiChatCompletions': { kind: 'stream' as const, frames: result.events }, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); @@ -298,7 +294,8 @@ const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage * usage chunk is asked for above everything, and coming back the vendor dialects have the * first say — the generic rules above them then read a body already in OpenAI-canonical form, * with the cache-bucket fold seeing cache fields under OpenAI's names and the carrier split - * seeing usage the fold has already settled. + * seeing usage the fold has already settled. The meter is above all of them, which is what + * makes the figure it bills the one the client is shown. * * `disableReasoningOnForcedToolChoice` and `stripPromptCacheKey` are here rather than above * the fork, because both speak about what an upstream's OpenAI Chat Completions endpoint accepts — @@ -307,6 +304,12 @@ const callOpenAIChatCompletionsUpstream = (streamedUsage: string) => defineStage * and the field an upstream would reject is gone before a vendor rewrites what is left. */ export const openaiChatCompletionsWire = (streamedUsage: string): readonly Stage[] => [ + meterChatWire({ + wire: 'openaiChatCompletions', + answer: 'response.chat.openaiChatCompletions', + streamedUsage, + read: meterOpenAIChatCompletions, + }), includeUsageStreamOptionsForOpenAIChatCompletions, normalizeUsageForOpenAIChatCompletions, disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, @@ -316,7 +319,7 @@ export const openaiChatCompletionsWire = (streamedUsage: string): readonly Stage vendorDeepSeekNormalizeForOpenAIChatCompletions, vendorQwenNormalizeForOpenAIChatCompletions, vendorKimiNormalizeForOpenAIChatCompletions, - callOpenAIChatCompletionsUpstream(streamedUsage), + callOpenAIChatCompletionsUpstream, ]; /** This family's own reading, which every wire under it hands up. */ diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index f0b4831878..8846f13d1f 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -82,6 +82,7 @@ import { vendorDeepSeekNormalizeForOpenAIResponses, vendorQwenNormalizeForOpenAIResponses, } from '../interceptors.ts'; +import { meterChatWire } from '../meter.ts'; import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import { applyRulesToUpstreamOpenAIResponses } from '../shared/alias-rules.ts'; import { tryCatchChatServeFailure } from '../shared/errors.ts'; @@ -344,16 +345,15 @@ const renderSSE = (frames: AsyncIterable defineStage< +const callOpenAIResponsesUpstream = defineStage< R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, - R<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'> & Record, + R<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'>, ChatServices >({ name: 'callOpenAIResponsesUpstream', return: { provides: [ 'response.chat.openaiResponses', - streamedUsage, 'response.usage.billable', 'response.http.headers', ], @@ -394,7 +394,6 @@ const callOpenAIResponsesUpstream = (streamedUsage: string) => defineStage< return move({ ...facts, 'response.chat.openaiResponses': { status: 502, message: error instanceof Error ? error.message : String(error) }, - [streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], }); @@ -417,7 +416,6 @@ const callOpenAIResponsesUpstream = (streamedUsage: string) => defineStage< message: text, ...(parsed === undefined ? {} : { body: parsed }), }, - [streamedUsage]: null, 'response.usage.billable': called, 'response.http.headers': [...result.response.headers], }); @@ -435,17 +433,14 @@ const callOpenAIResponsesUpstream = (streamedUsage: string) => defineStage< return move({ ...facts, 'response.chat.openaiResponses': { kind: 'value' as const, body: result.result }, - [streamedUsage]: null, 'response.usage.billable': [billedOpenAIResponsesEntity(identity, billableUsageFromOpenAIResponsesResult(result.result) ?? undefined)], 'response.http.headers': [], }); } - const metered = meterOpenAIResponses(result.events, identity, use.gateway.attempt); return move({ ...facts, - 'response.chat.openaiResponses': { kind: 'stream' as const, frames: metered.frames }, - [streamedUsage]: metered.outcome, + 'response.chat.openaiResponses': { kind: 'stream' as const, frames: result.events }, 'response.usage.billable': called, 'response.http.headers': [...(result.headers ?? new Headers())], }); @@ -509,10 +504,17 @@ export const openaiResponsesWireRules: readonly Stage[] = [ vendorQwenNormalizeForOpenAIResponses, ]; -/** The OpenAI Responses wire, as the chain that dials it. */ +/** The OpenAI Responses wire, as the chain that dials it. The meter is above every rule, which + * is what makes the figure it bills the one the client is shown. */ export const openaiResponsesWire = (streamedUsage: string): readonly Stage[] => [ + meterChatWire({ + wire: 'openaiResponses', + answer: 'response.chat.openaiResponses', + streamedUsage, + read: meterOpenAIResponses, + }), ...openaiResponsesWireRules, - callOpenAIResponsesUpstream(streamedUsage), + callOpenAIResponsesUpstream, ]; /** This family's own reading, which every wire under it hands up. */ From 2271d4929e37dcbea3aa68e21b7804a411ce29a8 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 08:49:46 +0800 Subject: [PATCH 088/156] refactor(chat): move the dormant shims out of the replaced surface's directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preparation, and no behaviour change: the server-tool shim, its two tools and their tests move from `openai-responses/interceptors/` to `openai-responses/server-tools/`, and Claude Code's probe predicate moves up beside the stage that reads it. What was left in `interceptors/` was the last thing pointing at a concept the chain replaced. The probe module also loses the interceptor it still exported. The stage in `pipeline.ts` had already taken over answering; only `isClaudeCodeProbe` and `probeFrames` were still reached, and its test drove the dead export rather than either. It drives the predicate now, which is what all thirteen of its cases were actually about. The one case that asserted on the interceptor's own result shape — no performance context, no final metadata — is a property of the stage rather than of a predicate, and moves with it in the port that follows. --- .../claude-code-probe_test.ts | 115 ++++++++++++++ .../answer-claude-code-probe_test.ts | 148 ------------------ .../image-generation-integration_test.ts | 14 +- .../server-tools/image-generation_test.ts | 10 +- .../shim_test.ts} | 10 +- .../server-tools/web-search_test.ts | 6 +- ...ude-code-probe.ts => claude-code-probe.ts} | 15 -- .../chat/anthropic-messages/pipeline.ts | 2 +- .../server-tools/image-generation.ts | 16 +- .../shim.ts} | 2 +- .../server-tools/web-search.ts | 20 +-- 11 files changed, 155 insertions(+), 203 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/anthropic-messages/claude-code-probe_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe_test.ts rename packages/gateway/__tests__/data-plane/chat/openai-responses/{interceptors => }/server-tools/image-generation-integration_test.ts (97%) rename packages/gateway/__tests__/data-plane/chat/openai-responses/{interceptors => }/server-tools/image-generation_test.ts (99%) rename packages/gateway/__tests__/data-plane/chat/openai-responses/{interceptors/server-tool-shim_test.ts => server-tools/shim_test.ts} (99%) rename packages/gateway/__tests__/data-plane/chat/openai-responses/{interceptors => }/server-tools/web-search_test.ts (98%) rename packages/gateway/src/data-plane/chat/anthropic-messages/{interceptors/answer-claude-code-probe.ts => claude-code-probe.ts} (86%) rename packages/gateway/src/data-plane/chat/openai-responses/{interceptors => }/server-tools/image-generation.ts (99%) rename packages/gateway/src/data-plane/chat/openai-responses/{interceptors/server-tool-shim.ts => server-tools/shim.ts} (99%) rename packages/gateway/src/data-plane/chat/openai-responses/{interceptors => }/server-tools/web-search.ts (97%) diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/claude-code-probe_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/claude-code-probe_test.ts new file mode 100644 index 0000000000..ec6587acd9 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/claude-code-probe_test.ts @@ -0,0 +1,115 @@ +import { test } from 'vitest'; + +import { isClaudeCodeProbe, probeFrames } from '../../../../src/data-plane/chat/anthropic-messages/claude-code-probe.ts'; +import { collectAnthropicMessagesProtocolEventsToResult, type AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { assert, assertEquals } from '@floway-dev/test-utils'; + +// The CLI emits `claude-cli/ (external, …)`, where the +// entrypoint substitutes for `cli` and up to three optional segments append — +// which is why the predicate anchors on the `claude-cli/` prefix only. +// This is the default entrypoint form. +const PROBE_USER_AGENT = 'claude-cli/2.1.226 (external, cli)'; + +// The shape of the 2.1.226 `/model` validation probe: one user turn holding +// one ephemeral text block, `max_tokens: 1`, no tools. The real body also +// carries the CLI's system-prompt array and a `betas` list, neither of which +// the predicate looks at. +const probePayload = (overrides: Partial = {}): AnthropicMessagesPayload => ({ + model: 'test-model', + max_tokens: 1, + system: [{ type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }], + messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi', cache_control: { type: 'ephemeral' } }] }], + metadata: { user_id: 'user_0_account__session_0' }, + ...overrides, +}); + +/** What the stage that answers this asks: is the turn in front of it the CLI's own probe? + * Everything below is that predicate, over the bodies the CLI has been observed to send. */ +const invocation = (payload: AnthropicMessagesPayload, userAgent: string | null = PROBE_USER_AGENT): boolean => + isClaudeCodeProbe(payload, new Headers(userAgent === null ? {} : { 'user-agent': userAgent })); + +const assertForwarded = (answered: boolean, message: string) => { assertEquals(answered, false, message); }; + +const assertAnswered = (answered: boolean, message: string) => { assertEquals(answered, true, message); }; + +test('answers the /model validation probe without dialing the upstream', async () => { + assertEquals(invocation(probePayload()), true); + + const message = await collectAnthropicMessagesProtocolEventsToResult(probeFrames('test-model')); + assertEquals(message.type, 'message'); + assertEquals(message.role, 'assistant'); + assertEquals(message.model, 'test-model'); + assertEquals(message.content, []); + assertEquals(message.stop_reason, 'max_tokens'); + assertEquals(message.usage.input_tokens, 0); + assertEquals(message.usage.output_tokens, 0); + assert(message.id.startsWith('msg_')); +}); + +test('answers every fixed probe prompt, in block and bare-string form', () => { + for (const prompt of ['Hi', 'test']) { + assertAnswered(invocation(probePayload({ messages: [{ role: 'user', content: prompt }] })), `bare string: ${prompt}`); + assertAnswered(invocation(probePayload({ messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }] })), `text block: ${prompt}`); + } +}); + +test('forwards the probe prompts the gateway cannot answer truthfully', () => { + // `quota` reads rate-limit response headers a synthesized turn cannot carry; + // `.` only ever reaches a Bedrock / Vertex / Mantle base URL; `hello` is a + // third-party report's reconstruction that no observed build sends. + for (const prompt of ['quota', '.', 'hello']) { + assertForwarded(invocation(probePayload({ messages: [{ role: 'user', content: prompt }] })), `prompt: ${prompt}`); + } +}); + +test('forwards a probe prompt whose casing the CLI has never been observed to send', () => { + assertForwarded(invocation(probePayload({ messages: [{ role: 'user', content: 'HI' }] })), 'uppercased'); +}); + +test('forwards a one-token request from a client that is not Claude Code', () => { + assertForwarded(invocation(probePayload(), 'python-httpx/0.28.1'), 'other client'); + assertForwarded(invocation(probePayload(), null), 'no user-agent'); +}); + +test('forwards a Claude Code turn whose output cap is not one token', () => { + assertForwarded(invocation(probePayload({ max_tokens: 32_000 })), 'max_tokens 32000'); +}); + +test('forwards a Claude Code turn that carries the session tools', () => { + assertForwarded(invocation(probePayload({ + tools: [{ name: 'Bash', input_schema: { type: 'object' } }], + })), 'tools present'); +}); + +test('forwards a conversation whose sole turn is not one of the fixed prompts', () => { + assertForwarded(invocation(probePayload({ + messages: [{ role: 'user', content: 'Hi, can you explain this file?' }], + })), 'real question'); +}); + +test('forwards a multi-turn conversation that happens to end on a probe prompt', () => { + assertForwarded(invocation(probePayload({ + messages: [ + { role: 'user', content: 'Hi' }, + { role: 'assistant', content: 'Hello!' }, + { role: 'user', content: 'Hi' }, + ], + })), 'three turns'); +}); + +test('forwards a sole turn that carries more than one block', () => { + assertForwarded(invocation(probePayload({ + messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }, { type: 'text', text: 'and explain this file' }] }], + })), 'two text blocks'); +}); + +test('forwards a sole turn that is not a user turn', () => { + assertForwarded(invocation(probePayload({ messages: [{ role: 'assistant', content: 'Hi' }] })), 'assistant role'); + assertForwarded(invocation(probePayload({ messages: [{ role: 'system', content: 'Hi' }] })), 'system role'); +}); + +test('forwards a turn whose sole block is not text', () => { + assertForwarded(invocation(probePayload({ + messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AA==' } }] }], + })), 'image block'); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe_test.ts deleted file mode 100644 index 72ff2aabf5..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe_test.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { test, vi } from 'vitest'; - -import { answerClaudeCodeProbe } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts'; -import type { AnthropicMessagesInvocation } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { collectAnthropicMessagesProtocolEventsToResult, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult } from '@floway-dev/provider'; -import { assert, assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -// The CLI emits `claude-cli/ (external, …)`, where the -// entrypoint substitutes for `cli` and up to three optional segments append — -// which is why the predicate anchors on the `claude-cli/` prefix only. -// This is the default entrypoint form. -const PROBE_USER_AGENT = 'claude-cli/2.1.226 (external, cli)'; - -// The shape of the 2.1.226 `/model` validation probe: one user turn holding -// one ephemeral text block, `max_tokens: 1`, no tools. The real body also -// carries the CLI's system-prompt array and a `betas` list, neither of which -// the predicate looks at. -const probePayload = (overrides: Partial = {}): AnthropicMessagesPayload => ({ - model: 'test-model', - max_tokens: 1, - system: [{ type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }], - messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi', cache_control: { type: 'ephemeral' } }] }], - metadata: { user_id: 'user_0_account__session_0' }, - ...overrides, -}); - -const invocation = (payload: AnthropicMessagesPayload, userAgent: string | null = PROBE_USER_AGENT): AnthropicMessagesInvocation => ({ - payload, - candidate: stubModelCandidate({ model: { endpoints: { openaiResponses: {} } } }), - targetApi: 'openaiResponses', - headers: new Headers(userAgent === null ? {} : { 'user-agent': userAgent }), -}); - -const passthrough = async (): Promise>> => - eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity); - -const runProbe = async (input: AnthropicMessagesInvocation) => { - const run = vi.fn(passthrough); - const result = await answerClaudeCodeProbe(input, stubCtx, run); - return { result, run }; -}; - -const assertForwarded = async (input: AnthropicMessagesInvocation, message: string) => { - const { run } = await runProbe(input); - assertEquals(run.mock.calls.length, 1, message); -}; - -const assertAnswered = async (input: AnthropicMessagesInvocation, message: string) => { - const { run } = await runProbe(input); - assertEquals(run.mock.calls.length, 0, message); -}; - -test('answers the /model validation probe without dialing the upstream', async () => { - const { result, run } = await runProbe(invocation(probePayload())); - - assertEquals(run.mock.calls.length, 0); - assert(result.type === 'events'); - const message = await collectAnthropicMessagesProtocolEventsToResult(result.events); - assertEquals(message.type, 'message'); - assertEquals(message.role, 'assistant'); - assertEquals(message.model, 'test-model'); - assertEquals(message.content, []); - assertEquals(message.stop_reason, 'max_tokens'); - assertEquals(message.usage.input_tokens, 0); - assertEquals(message.usage.output_tokens, 0); - assert(message.id.startsWith('msg_')); -}); - -test('reports no performance context so the turn contributes no latency sample', async () => { - const { result } = await runProbe(invocation(probePayload())); - - assert(result.type === 'events'); - assertEquals(result.performance, undefined); - assertEquals(result.finalMetadata, undefined); -}); - -test('answers every fixed probe prompt, in block and bare-string form', async () => { - for (const prompt of ['Hi', 'test']) { - await assertAnswered(invocation(probePayload({ messages: [{ role: 'user', content: prompt }] })), `bare string: ${prompt}`); - await assertAnswered(invocation(probePayload({ messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }] })), `text block: ${prompt}`); - } -}); - -test('forwards the probe prompts the gateway cannot answer truthfully', async () => { - // `quota` reads rate-limit response headers a synthesized turn cannot carry; - // `.` only ever reaches a Bedrock / Vertex / Mantle base URL; `hello` is a - // third-party report's reconstruction that no observed build sends. - for (const prompt of ['quota', '.', 'hello']) { - await assertForwarded(invocation(probePayload({ messages: [{ role: 'user', content: prompt }] })), `prompt: ${prompt}`); - } -}); - -test('forwards a probe prompt whose casing the CLI has never been observed to send', async () => { - await assertForwarded(invocation(probePayload({ messages: [{ role: 'user', content: 'HI' }] })), 'uppercased'); -}); - -test('forwards a one-token request from a client that is not Claude Code', async () => { - await assertForwarded(invocation(probePayload(), 'python-httpx/0.28.1'), 'other client'); - await assertForwarded(invocation(probePayload(), null), 'no user-agent'); -}); - -test('forwards a Claude Code turn whose output cap is not one token', async () => { - await assertForwarded(invocation(probePayload({ max_tokens: 32_000 })), 'max_tokens 32000'); -}); - -test('forwards a Claude Code turn that carries the session tools', async () => { - await assertForwarded(invocation(probePayload({ - tools: [{ name: 'Bash', input_schema: { type: 'object' } }], - })), 'tools present'); -}); - -test('forwards a conversation whose sole turn is not one of the fixed prompts', async () => { - await assertForwarded(invocation(probePayload({ - messages: [{ role: 'user', content: 'Hi, can you explain this file?' }], - })), 'real question'); -}); - -test('forwards a multi-turn conversation that happens to end on a probe prompt', async () => { - await assertForwarded(invocation(probePayload({ - messages: [ - { role: 'user', content: 'Hi' }, - { role: 'assistant', content: 'Hello!' }, - { role: 'user', content: 'Hi' }, - ], - })), 'three turns'); -}); - -test('forwards a sole turn that carries more than one block', async () => { - await assertForwarded(invocation(probePayload({ - messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }, { type: 'text', text: 'and explain this file' }] }], - })), 'two text blocks'); -}); - -test('forwards a sole turn that is not a user turn', async () => { - await assertForwarded(invocation(probePayload({ messages: [{ role: 'assistant', content: 'Hi' }] })), 'assistant role'); - await assertForwarded(invocation(probePayload({ messages: [{ role: 'system', content: 'Hi' }] })), 'system role'); -}); - -test('forwards a turn whose sole block is not text', async () => { - await assertForwarded(invocation(probePayload({ - messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AA==' } }] }], - })), 'image block'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts similarity index 97% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts index 61ece2aff8..737d9baca1 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts @@ -1,9 +1,9 @@ import { beforeEach, test, vi } from 'vitest'; -import type { OpenAIResponsesInvocation } from '../../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { initRepo } from '../../../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../../../test-utils/gateway-ctx.ts'; +import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; +import { initRepo } from '../../../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../../../repo/memory.ts'; +import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; import { createInMemoryImageProcessor, initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; import { eventFrame } from '@floway-dev/protocols/common'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; @@ -75,7 +75,7 @@ const defaultCandidates = vi.hoisted(() => () => [{ fetcher: (request: Request) => fetch(request), }]); -vi.mock('../../../../../../src/data-plane/providers/resolution.ts', () => ({ +vi.mock('../../../../../src/data-plane/providers/resolution.ts', () => ({ enumerateModelCandidates: vi.fn(async () => { const override = stub.nextResolutionOverride; if (override !== null) { @@ -87,8 +87,8 @@ vi.mock('../../../../../../src/data-plane/providers/resolution.ts', () => ({ })); // Imported AFTER vi.mock so the mocked registry is in effect. -const { withOpenAIResponsesServerToolShim } = await import('../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'); -const { imageGenerationServerTool } = await import('../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts'); +const { withOpenAIResponsesServerToolShim } = await import('../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'); +const { imageGenerationServerTool } = await import('../../../../../src/data-plane/chat/openai-responses/server-tools/image-generation.ts'); const shim = withOpenAIResponsesServerToolShim([imageGenerationServerTool]); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation_test.ts similarity index 99% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation_test.ts index cdecfa15d6..2311d5ffe8 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation_test.ts @@ -1,5 +1,6 @@ import { beforeEach, test } from 'vitest'; +import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; import { buildImageGenerationFunctionTool, createImageSourceInspector, @@ -17,11 +18,10 @@ import { SHIM_TOOL_NAME, synthesizeImageGenerationCallId, transformInputItemsForImageGeneration, -} from '../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { initRepo } from '../../../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../../../test-utils/gateway-ctx.ts'; +} from '../../../../../src/data-plane/chat/openai-responses/server-tools/image-generation.ts'; +import { initRepo } from '../../../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../../../repo/memory.ts'; +import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; import { initExternalResourceFetcher } from '@floway-dev/platform'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputImage, OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesTool } from '@floway-dev/protocols/openai-responses'; import { assert, assertEquals, assertFalse, assertStringIncludes, assertThrows, stubModelCandidate } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts similarity index 99% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts index b86b23fe24..8cc770aa3c 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts @@ -1,6 +1,8 @@ import { beforeEach, test, vi } from 'vitest'; -import { withOpenAIResponsesServerToolShim } from '../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'; +import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; +import { createNonOpenAIResponsesSourceStore } from '../../../../../src/data-plane/chat/openai-responses/items/store.ts'; +import { withOpenAIResponsesServerToolShim } from '../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'; import { consumeTurnStreaming, createMergeState, @@ -11,10 +13,8 @@ import { type ServerToolResultSlot, type TurnSummary, type UpstreamTerminal, -} from '../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'; -import { SHIM_TOOL_NAME, webSearchServerTool } from '../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts'; -import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { createNonOpenAIResponsesSourceStore } from '../../../../../src/data-plane/chat/openai-responses/items/store.ts'; +} from '../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'; +import { SHIM_TOOL_NAME, webSearchServerTool } from '../../../../../src/data-plane/chat/openai-responses/server-tools/web-search.ts'; import type { ChatGatewayCtx } from '../../../../../src/data-plane/chat/shared/gateway-ctx.ts'; import { resolveAlphaSearchDispatcher } from '../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; import type { AlphaSearchDispatcher } from '../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/web-search_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/web-search_test.ts similarity index 98% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/web-search_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/web-search_test.ts index cf64eea9fc..f78809a987 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/web-search_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/web-search_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { resolveServerToolName } from '../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'; +import { resolveServerToolName } from '../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'; import { alphaSearchSettingsFromHosted, buildShimFunctionTool, @@ -11,8 +11,8 @@ import { transformInputItemsForWebSearch, WEB_SEARCH_HOSTED_TYPES, type WebSearchCallPrivatePayload, -} from '../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts'; -import { assertLocalWebSearchSupport, findMatches, formatMatches, isUrlAllowed, parseWebSearchOperations, type WebSearchOperation } from '../../../../../../src/data-plane/tools/web-search/operations.ts'; +} from '../../../../../src/data-plane/chat/openai-responses/server-tools/web-search.ts'; +import { assertLocalWebSearchSupport, findMatches, formatMatches, isUrlAllowed, parseWebSearchOperations, type WebSearchOperation } from '../../../../../src/data-plane/tools/web-search/operations.ts'; import type { OpenAIResponsesHostedTool, OpenAIResponsesTool, OpenAIResponsesWebSearchAction, OpenAIResponsesWebSearchResult } from '@floway-dev/protocols/openai-responses'; import { assert, assertEquals, assertThrows } from '@floway-dev/test-utils'; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts similarity index 86% rename from packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts rename to packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts index 05fc86ecb7..f42b4a7a8b 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts @@ -1,8 +1,5 @@ -import type { AnthropicMessagesInterceptor } from './types.ts'; -import { telemetryModelIdentity } from '../../../shared/telemetry/attribution.ts'; import { generateAnthropicId, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventResult, providerModelOf, type ExecuteResult, type ModelCandidate } from '@floway-dev/provider'; // Claude Code answers "is this model usable?" by generating one token against // it. `/model ` runs the CLI's `model_validation` side query — a @@ -113,15 +110,3 @@ export const probeFrames = async function* (model: string): AsyncGenerator> => - eventResult(probeFrames(model), telemetryModelIdentity(candidate, providerModelOf(candidate).id)); - -export const answerClaudeCodeProbe: AnthropicMessagesInterceptor = async (ctx, _gatewayCtx, run) => - isClaudeCodeProbe(ctx.payload, ctx.headers) - ? probeResult(ctx.candidate, ctx.payload.model) - : await run(); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 5d1e4504e0..7fd68af931 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -22,8 +22,8 @@ import { wrapAnthropicMessagesAffinityEgress } from './affinity/egress.ts'; import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; +import { isClaudeCodeProbe, probeFrames } from './claude-code-probe.ts'; import { renderAnthropicMessagesError } from './errors.ts'; -import { isClaudeCodeProbe, probeFrames } from './interceptors/answer-claude-code-probe.ts'; import { createAnthropicMessagesBillableUsageReader } from './usage.ts'; import { recordStream, streamReferenceOf } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts similarity index 99% rename from packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts rename to packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts index 211a5f7c29..4a6c6bb81a 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts @@ -1,11 +1,11 @@ -import { sleep } from '../../../../../shared/sleep.ts'; -import { enumerateModelCandidates } from '../../../../providers/resolution.ts'; -import { appendFailedUpstreams } from '../../../../shared/failed-upstreams.ts'; -import { stampUpstreamCallStart, type AttemptState } from '../../../../shared/gateway-ctx.ts'; -import { recordPerformance, type PerformanceTelemetryContext } from '../../../../shared/telemetry/performance.ts'; -import { recordTokenUsage, tokenUsageFromOpenAIImagesBody } from '../../../../shared/telemetry/usage.ts'; -import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../../shared/external-image-loader.ts'; -import type { ServerToolLifecycleEvent, ServerToolOutputItem, ServerToolRegistration, ServerToolTerminal } from '../server-tool-shim.ts'; +import type { ServerToolLifecycleEvent, ServerToolOutputItem, ServerToolRegistration, ServerToolTerminal } from './shim.ts'; +import { sleep } from '../../../../shared/sleep.ts'; +import { enumerateModelCandidates } from '../../../providers/resolution.ts'; +import { appendFailedUpstreams } from '../../../shared/failed-upstreams.ts'; +import { stampUpstreamCallStart, type AttemptState } from '../../../shared/gateway-ctx.ts'; +import { recordPerformance, type PerformanceTelemetryContext } from '../../../shared/telemetry/performance.ts'; +import { recordTokenUsage, tokenUsageFromOpenAIImagesBody } from '../../../shared/telemetry/usage.ts'; +import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../shared/external-image-loader.ts'; import { dimensionsFromBytes, getImageProcessor, type BackgroundScheduler } from '@floway-dev/platform'; import { decodeForgivingBase64, encodeHex, isImageMediaType, mediaTypeEssence, parseSSEStream } from '@floway-dev/protocols/common'; import { diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/shim.ts similarity index 99% rename from packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts rename to packages/gateway/src/data-plane/chat/openai-responses/server-tools/shim.ts index f87f1d6c54..95859e2001 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/shim.ts @@ -1,8 +1,8 @@ import { jsonrepair } from 'jsonrepair'; -import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from './types.ts'; import { truncatePreservingCodePoints } from '../../../shared/text.ts'; import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; +import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from '../interceptors/types.ts'; import type { StatefulOpenAIResponsesStore } from '../items/store.ts'; import type { InterceptorRun } from '@floway-dev/interceptor'; import { eventFrame, sumBillableUsage, type ProtocolFrame } from '@floway-dev/protocols/common'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/web-search.ts similarity index 97% rename from packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts rename to packages/gateway/src/data-plane/chat/openai-responses/server-tools/web-search.ts index 363f922483..95b921e5eb 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/web-search.ts @@ -1,9 +1,10 @@ -import { shortId } from '../../../../../shared/short-id.ts'; -import { truncatePreservingCodePoints } from '../../../../shared/text.ts'; -import { executeAlphaSearch } from '../../../../tools/web-search/alpha-search/execution.ts'; -import { resolveAlphaSearchDispatcher } from '../../../../tools/web-search/alpha-search/upstream.ts'; -import { loadWebSearchConfig } from '../../../../tools/web-search/config.ts'; -import { normalizeDomainEntry } from '../../../../tools/web-search/domain-normalize.ts'; +import { type ServerToolLoopState, type ServerToolOutputItem, type ServerToolRegistration } from './shim.ts'; +import { shortId } from '../../../../shared/short-id.ts'; +import { truncatePreservingCodePoints } from '../../../shared/text.ts'; +import { executeAlphaSearch } from '../../../tools/web-search/alpha-search/execution.ts'; +import { resolveAlphaSearchDispatcher } from '../../../tools/web-search/alpha-search/upstream.ts'; +import { loadWebSearchConfig } from '../../../tools/web-search/config.ts'; +import { normalizeDomainEntry } from '../../../tools/web-search/domain-normalize.ts'; import { actionSearchQueries, assertLocalWebSearchSupport, @@ -24,10 +25,9 @@ import { type WebSearchExecutionSession, type WebSearchFilters, type WebSearchOperation, -} from '../../../../tools/web-search/operations.ts'; -import { resolveConfiguredWebSearchProvider } from '../../../../tools/web-search/provider.ts'; -import type { ConfiguredWebSearchProvider } from '../../../../tools/web-search/types.ts'; -import { type ServerToolLoopState, type ServerToolOutputItem, type ServerToolRegistration } from '../server-tool-shim.ts'; +} from '../../../tools/web-search/operations.ts'; +import { resolveConfiguredWebSearchProvider } from '../../../tools/web-search/provider.ts'; +import type { ConfiguredWebSearchProvider } from '../../../tools/web-search/types.ts'; import type { OpenAIResponsesFunctionTool, OpenAIResponsesFunctionToolCallItem, OpenAIResponsesHostedTool, OpenAIResponsesInputItem, OpenAIResponsesOutputWebSearchCall, OpenAIResponsesTool, OpenAIResponsesWebSearchAction } from '@floway-dev/protocols/openai-responses'; import { createRandomOpenAIResponsesItemId, WEB_SEARCH_HOSTED_TYPE_NAMES } from '@floway-dev/protocols/openai-responses'; import { providerModelOf } from '@floway-dev/provider'; From 5eae693698e92522bd0b233155daab617767aaba Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 09:27:16 +0800 Subject: [PATCH 089/156] feat(openai-responses): bring the server-tool shim back, as the stage the ruling describes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hosted `web_search` or `image_generation` tool declared against an upstream that does not implement it was reaching that upstream unshimmed. The shim had never become a stage, so it and the two tools under it were reachable only from their own tests, and the two flags that gate them did nothing. It is one stage, because what it does is one thing: a **fork it merges**. The hosted tool is rewritten into a function tool on the way down, the model's call to it is executed here, the result is folded back into the input, and the upstream is asked again — until the model stops calling it. Every ask is another descent, and the frames of all of them are spliced into one stream under one synthesized response id, so nothing above sees the repetition. That is exactly what `writeSettlement` meant by "repetition passes through the stage that observes usage"; this is that stage. It sits directly above the dial and below the fork, so a descent re-dials the same candidate and an upstream that refuses mid-loop rides up to `failover`, which re-runs the whole loop against the next candidate rather than resuming a half-finished one. What each descent billed travels up separately and is **concatenated**, not summed. The interceptor summed into one figure; each turn is its own upstream call with its own pricing facts — a service tier and an input size a later turn need not share — so the run settles one row per call rather than one row whose rate is a guess. That is the one deliberate behaviour change, and the two tests that read the interceptor's own metadata now read the reading the stage hands up: one asserts both turns' identities are named in the order they were called, the other that the cost lands where settlement writes from. The registrations keep their interface: they read a body and a candidate, and the object they read it through is the stage's own, built per attempt and handed nowhere. Their 373 tests kept every assertion — what changed is the driver, which scripts the *ending* rather than a `run` callback and lets the real chain run above it. A route test proves it live: a client declaring a hosted image-generation tool against a Copilot upstream now reaches that upstream with the shim's function tool, where before the boundary simply dropped the hosted one. --- .../openai-responses/server-tools/drive.ts | 110 +++++ .../image-generation-integration_test.ts | 4 +- .../server-tools/shim_test.ts | 30 +- .../chat/routes-openai-responses_test.ts | 13 +- .../chat/openai-responses/pipeline.ts | 10 + .../openai-responses/server-tools/shim.ts | 266 +----------- .../openai-responses/server-tools/stage.ts | 401 ++++++++++++++++++ 7 files changed, 566 insertions(+), 268 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/server-tools/stage.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts new file mode 100644 index 0000000000..872fbba320 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts @@ -0,0 +1,110 @@ +// Driving the server-tool stage from a test that scripts the dial. +// +// The stage descends once per upstream turn, so what a test scripts is the *ending* — one +// answer per descent, in the order the loop asks for them. Everything above the ending is the +// real chain: the registrations prepare, the loop splices, and what comes back is read off the +// facts rather than out of a mock. +// +// `invocation.payload` is written back on every descent, so a test that asserts on what the +// upstream was last asked for reads the same field it always did. + +import type { ServerToolRegistration } from '../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'; +import { runOpenAIResponsesServerTools } from '../../../../../src/data-plane/chat/openai-responses/server-tools/stage.ts'; +import type { ChatGatewayCtx } from '../../../../../src/data-plane/chat/shared/gateway-ctx.ts'; +import type { StreamOutcome } from '../../../../../src/data-plane/pipeline/serve.ts'; +import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../../../../src/data-plane/shared/telemetry/usage.ts'; +import { compose, defer, defineStage, move, run, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { EventResultMetadata, ExecuteResult, ModelCandidate, OpenAIResponsesInvocation } from '@floway-dev/provider'; + +const ANSWER = 'response.chat.openaiResponses'; +const STREAMED_USAGE = 'response.chat.openaiResponses.streamedUsage'; + +type Result = ExecuteResult>; + +/** What a scripted turn will have billed, as the wire's meter would have handed it up: one + * entity naming the model that answered, carrying what it reported. A turn that reports + * nothing still names its identity, which is how the loop's own reading shows which upstream + * each of its calls reached. */ +const readingOf = (result: Extract) => defer( + (result.finalMetadata ?? Promise.resolve({ modelIdentity: result.modelIdentity, billableUsage: undefined })) + .then((metadata: EventResultMetadata): StreamOutcome => { + const tokens = tokenUsageFromBillableUsage(metadata.billableUsage); + return { + billable: [{ identity: metadata.modelIdentity, quantities: tokens === null ? {} : tokenUsageMeasurement(tokens).quantities }], + failed: false, + }; + }), +); + +/** A scripted refusal as the failure value a chain carries. An internal error has no body of + * its own, so what travels is the sentence it raised. */ +const failureOf = (result: Exclude) => ({ + status: result.type === 'internal-error' ? 500 : result.status, + message: result.type === 'internal-error' ? result.error.message : new TextDecoder().decode(result.body), +}); + +/** What the run answered with, in the shape the assertions read. */ +const resultOf = (facts: Record): Result => { + const answer = facts[ANSWER] as { status?: number; message?: string; kind?: string; frames?: unknown }; + if (answer.kind === 'stream') { + return { type: 'events', events: answer.frames as AsyncIterable>, modelIdentity: undefined } as unknown as Result; + } + return { + type: 'api-error', + source: 'gateway', + status: answer.status!, + headers: new Headers({ 'content-type': 'application/json' }), + body: new TextEncoder().encode(answer.message!), + } as Result; +}; + +export const driveServerToolStage = (registrations: readonly ServerToolRegistration[]) => + async ( + invocation: OpenAIResponsesInvocation, + gatewayCtx: ChatGatewayCtx, + dial: () => Promise, + ): Promise | null }> => { + const ending = defineStage, Record>({ + name: 'scriptedDial', + return: { provides: [ANSWER, STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'] }, + execute: async facts => { + invocation.payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const result = await dial(); + const answered = result.type === 'events' + ? { [ANSWER]: { kind: 'stream' as const, frames: result.events }, [STREAMED_USAGE]: readingOf(result) } + : { [ANSWER]: failureOf(result), [STREAMED_USAGE]: null }; + return move({ + ...facts, + ...answered, + 'response.usage.billable': result.type === 'events' + ? [{ identity: result.modelIdentity, quantities: {} }] + : [], + 'response.http.headers': [], + }); + }, + }); + + const chain = compose, Record>('serverToolsUnderTest', [ + runOpenAIResponsesServerTools(registrations, { + streamedUsage: STREAMED_USAGE, + targetOf: () => invocation.targetApi, + }) as never, + ending, + ]); + + const { facts } = await run(chain, move({ + 'request.chat.openaiResponses': invocation.payload, + 'route.attempt': { upstreamId: 'up_test', modelId: invocation.payload.model, flags: [] }, + 'ingress.http.headers': [...invocation.headers], + }), { + gateway: gatewayCtx, + background: () => {}, + resolveAttempt: (): ModelCandidate => invocation.candidate, + } as never); + + // The reading the run hands up, which is where every turn's identity and cost land now + // that a stage rather than an interceptor observes them. + return { ...resultOf(facts), reading: facts[STREAMED_USAGE] as Deferred | null }; + }; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts index 737d9baca1..8017b2460c 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts @@ -1,5 +1,6 @@ import { beforeEach, test, vi } from 'vitest'; +import { driveServerToolStage } from './drive.ts'; import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; import { initRepo } from '../../../../../src/repo/index.ts'; import { InMemoryRepo } from '../../../../repo/memory.ts'; @@ -87,10 +88,9 @@ vi.mock('../../../../../src/data-plane/providers/resolution.ts', () => ({ })); // Imported AFTER vi.mock so the mocked registry is in effect. -const { withOpenAIResponsesServerToolShim } = await import('../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'); const { imageGenerationServerTool } = await import('../../../../../src/data-plane/chat/openai-responses/server-tools/image-generation.ts'); -const shim = withOpenAIResponsesServerToolShim([imageGenerationServerTool]); +const shim = driveServerToolStage([imageGenerationServerTool]); const MODEL_IDENTITY = { model: 'orchestrator', upstream: 'u', modelKey: 'orchestrator', pricing: null }; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts index 8cc770aa3c..d1f61b1a27 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts @@ -1,8 +1,8 @@ import { beforeEach, test, vi } from 'vitest'; +import { driveServerToolStage } from './drive.ts'; import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; import { createNonOpenAIResponsesSourceStore } from '../../../../../src/data-plane/chat/openai-responses/items/store.ts'; -import { withOpenAIResponsesServerToolShim } from '../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'; import { consumeTurnStreaming, createMergeState, @@ -49,7 +49,7 @@ import type { import { type EventResult, type ExecuteResult, type FlagId } from '@floway-dev/provider'; import { assert, assertEquals, assertFalse, stubModelCandidate } from '@floway-dev/test-utils'; -const withOpenAIResponsesWebSearchShim = withOpenAIResponsesServerToolShim([webSearchServerTool]); +const withOpenAIResponsesWebSearchShim = driveServerToolStage([webSearchServerTool]); const emptyResult = (id: string, status: OpenAIResponsesResult['status']): OpenAIResponsesResult => ({ id, @@ -992,7 +992,7 @@ test('fetchPage whole-batch failure surfaces the open-page error text', async () // ── Domain filter input validation ──────────────────────────────────── test('invalid request registration preserves an upstream error type and null code', async () => { - const shim = withOpenAIResponsesServerToolShim([() => ({ + const shim = driveServerToolStage([() => ({ type: 'invalid-request', message: 'native image error', param: 'input', @@ -2872,11 +2872,13 @@ test('finalMetadata resolves with the LATEST turn modelIdentity, not turn 1', as const result = await shim(inv, makeGatewayCtx(), run); assert(result.type === 'events'); - // Drain so the multi-turn loop completes and finalMetadata resolves. + // Drain so the multi-turn loop completes and the reading it hands up settles. await collectFrames(result.events); - assert(result.finalMetadata !== undefined); - const meta = await result.finalMetadata; - assertEquals(meta.modelIdentity.modelKey, 'turn-2-key'); + assert(result.reading !== null); + const reading = await result.reading; + // Every call the loop made is its own billable entity, in the order it made them, so the + // variant a later turn was served by is named rather than frozen to the first one's. + assertEquals(reading.billable.map(entity => entity.identity.modelKey), ['turn-1-key', 'turn-2-key']); }); // ── Results are always included on web_search_call items ───────────── @@ -6238,10 +6240,10 @@ test('upstream that does not echo `tools` produces a synthesized envelope withou // ── Billable usage ──────────────────────────────────────────────────────── -test('the shim carries the upstream turn cost onto the metadata pricing reads', async () => { - // The shim publishes its own finalMetadata, and that is what pricing reads. - // Failing to carry the cost onto it loses billing for the whole response - // silently, because the client-facing usage is no longer consulted. +test('the shim carries the upstream turn cost onto the reading settlement writes from', async () => { + // The stage publishes its own reading, and that is what settlement writes the row from. + // Failing to carry the cost onto it loses billing for the whole response silently, because + // the client-facing usage is no longer consulted. const billableUsage: BillableUsage = { input: 10, cacheRead: 3, cacheWrite: 0, cacheWrite1h: 0, output: 2 }; makeStubDeps(); const inv = makeInvocation({ enabledFlags: new Set(['responses-web-search-shim'] as const) }); @@ -6257,7 +6259,11 @@ test('the shim carries the upstream turn cost onto the metadata pricing reads', assert(result.type === 'events'); await collectFrames(result.events); - assertEquals((await result.finalMetadata!).billableUsage, billableUsage); + assert(result.reading !== null); + assertEquals((await result.reading).billable, [{ + identity: testTelemetryModelIdentity, + quantities: { input_tokens: '10', input_cache_read_tokens: '3', output_tokens: '2' }, + }]); }); test('consumeTurn forwards an event carrying no output_index instead of dropping it', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts index 2c42260609..fed12a48bf 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts @@ -97,11 +97,16 @@ test('a turn reaches Copilot over its own OpenAI Responses endpoint and comes ba expect(body.output.find(item => item.type === 'message')?.content?.[0]?.text).toBe(ANSWER); expect(body.usage).toMatchObject({ input_tokens: 11, output_tokens: 7 }); - // What the boundary made of the turn, on the wire: the tool its endpoint rejects is gone, - // the description it needs is filled, and neither field it refuses travels. - const tools = sent?.tools as { type: string; description?: string }[]; - expect(tools.map(tool => tool.type)).toEqual(['namespace']); + // What the turn looked like on the wire. The hosted image-generation tool never reached the + // Copilot boundary that would have dropped it: Copilot carries the + // `responses-image-generation-shim` flag, so the server-tool stage got there first and sent a + // function tool the model can call and this gateway executes. What the boundary did make of + // the rest is here — the description its namespace tool needs is filled, and neither field it + // refuses travels. + const tools = sent?.tools as { type: string; name?: string; description?: string }[]; + expect(tools.map(tool => tool.type)).toEqual(['namespace', 'function']); expect(tools[0]!.description).toBe('Tools in the repo namespace.'); + expect(tools[1]!.name).toBe('image_generation'); expect(sent).not.toHaveProperty('service_tier'); expect(sent).toMatchObject({ store: false }); }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 9167ecec25..46703b6ad1 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -59,6 +59,9 @@ import { hydrateOpenAIResponsesPayload } from './items/hydrate.ts'; import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; import { syntheticEventsFromResult } from './items/output.ts'; import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve-prep.ts'; +import { imageGenerationServerTool } from './server-tools/image-generation.ts'; +import { runOpenAIResponsesServerTools } from './server-tools/stage.ts'; +import { webSearchServerTool } from './server-tools/web-search.ts'; import { billableUsageFromOpenAIResponsesEvent, billableUsageFromOpenAIResponsesResult } from './usage.ts'; import { recordStream, streamReferenceOf } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; @@ -993,6 +996,13 @@ export const openaiResponsesServePipeline = ( beginStoredAttempt, expandShimCompactions, summarizeForCompaction(asksForCompaction), + // Directly above the dial, because every descent it makes is another dial of this same + // candidate: a hosted tool the upstream does not implement is emulated by asking again with + // the tool's result folded back in, and the frames of every ask are spliced into one turn. + runOpenAIResponsesServerTools([webSearchServerTool, imageGenerationServerTool], { + streamedUsage: OPENAI_RESPONSES_STREAMED_USAGE, + targetOf: candidate => openaiResponsesTarget.pick(candidate.model.endpoints), + }), dialChatWire({ source: 'request.chat.openaiResponses', needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/shim.ts index 95859e2001..92badfbb3a 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/shim.ts @@ -2,13 +2,10 @@ import { jsonrepair } from 'jsonrepair'; import { truncatePreservingCodePoints } from '../../../shared/text.ts'; import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; -import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from '../interceptors/types.ts'; import type { StatefulOpenAIResponsesStore } from '../items/store.ts'; -import type { InterceptorRun } from '@floway-dev/interceptor'; -import { eventFrame, sumBillableUsage, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { createRandomOpenAIResponsesItemId, - type CanonicalOpenAIResponsesPayload, type OpenAIResponsesFunctionTool, type OpenAIResponsesHostedTool, type OpenAIResponsesInputItem, @@ -18,7 +15,7 @@ import { type OpenAIResponsesTool, type OpenAIResponsesToolChoice, } from '@floway-dev/protocols/openai-responses'; -import type { EventResultMetadata, ExecuteResult } from '@floway-dev/provider'; +import type { OpenAIResponsesInvocation } from '@floway-dev/provider'; export interface MergeUsage { input_tokens?: number; @@ -128,7 +125,7 @@ export type ServerToolPrepareResult = export type ServerToolRegistration = (invocation: OpenAIResponsesInvocation, gatewayCtx: ChatGatewayCtx) => ServerToolPrepareResult | Promise; -type ActiveServerTool = Extract & { +export type ActiveServerTool = Extract & { toolName: string; // Absent only for replay activation; otherwise drives `tools` echo restore. canonicalHostedTool: OpenAIResponsesHostedTool | undefined; @@ -155,8 +152,6 @@ export interface TurnSummary { terminalStatus: UpstreamTerminal; } -type LatestUpstreamMetadata = Pick; - export const createMergeState = (): MergeState => ({ sequenceNumber: 0, outputIndex: 0, @@ -224,7 +219,7 @@ const usageOf = (usage: OpenAIResponsesResult['usage']): MergeUsage => { return out; }; -const rewriteHostedToolChoice = ( +export const rewriteHostedToolChoice = ( toolChoice: OpenAIResponsesToolChoice | null | undefined, active: readonly ActiveServerTool[], ): OpenAIResponsesToolChoice | null | undefined => { @@ -284,7 +279,7 @@ export const resolveServerToolName = (baseName: string, tools: readonly OpenAIRe // configuration. The replacement occupies the first declaration's array slot // so unrelated tools retain their relative order. // https://github.com/Menci/Floway/pull/172#issuecomment-4971739422 -const rewriteToolsForHostedShim = ( +export const rewriteToolsForHostedShim = ( tools: readonly OpenAIResponsesTool[], hosted: ServerToolHostedDispatch, toolName: string, @@ -442,7 +437,7 @@ const serverToolEndFrames = ( return frames; }; -const transformServerToolItems = ( +export const transformServerToolItems = ( items: OpenAIResponsesInputItem[], active: readonly ActiveServerTool[], ): OpenAIResponsesInputItem[] => { @@ -727,11 +722,15 @@ export const consumeTurnStreaming = async function* ( const MAX_BODY_EXCERPT_CHARS = 512; -const buildErrorFromResult = ( - result: Exclude, { type: 'events' }>, +/** What a refusal mid-loop becomes on the synthesized terminal. An upstream that wrote an + * OpenAI-shaped envelope is quoted verbatim — its own code, type and sentence — because that is + * what a client reads; anything else is reported as the status plus an excerpt of what came. */ +export const buildErrorFromRefusal = ( + status: number, + body: string, ): NonNullable => { - if (result.type === 'internal-error') return { message: result.error.message, code: 'server_error' }; - const decoded = new TextDecoder('utf-8', { fatal: false }).decode(result.body); + const result = { status, body }; + const decoded = body; let parsed: unknown = undefined; try { parsed = JSON.parse(decoded); @@ -756,29 +755,6 @@ const buildErrorFromResult = ( }; }; -const invalidRequestEnvelope = ( - message: string, - param: string | null, - code: string | null | undefined, - errorType = 'invalid_request_error', -): ExecuteResult> => { - const body = JSON.stringify({ - error: { - message, - type: errorType, - param, - code: code === undefined ? 'invalid_request_error' : code, - }, - }); - return { - type: 'api-error', - source: 'gateway', - status: 400, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(body), - }; -}; - // The terminal the shim emits downstream. Unlike `UpstreamTerminal` // (what we observed), this carries only the already-extracted `error` / // `incompleteDetails` the synthesized envelope needs; the output and @@ -795,7 +771,7 @@ const SYNTHESIZED_TERMINAL_FRAME: Record, merge: MergeState, store: StatefulOpenAIResponsesStore, @@ -860,213 +836,3 @@ async function* materializeServerToolItems( } } } - -// `finalMetadata` resolves when that turn's stream ends; the shim has already -// drained it before reaching here, so awaiting it cannot deadlock. -const accumulateBillableUsage = async ( - metadata: LatestUpstreamMetadata, - result: Extract>, { type: 'events' }>, -): Promise => { - const turn = result.finalMetadata === undefined ? undefined : (await result.finalMetadata).billableUsage; - metadata.billableUsage = sumBillableUsage(metadata.billableUsage, turn); -}; - -async function* runMultiTurnLoop(args: { - ctx: OpenAIResponsesInvocation; - run: InterceptorRun>>; - merge: MergeState; - loopState: ServerToolLoopState; - demoteForcedServerToolChoiceAfterFirstTurn: boolean; - turn1Iter: AsyncGenerator, TurnSummary>; - firstResult: Extract>, { type: 'events' }>; - dispatchers: ReadonlyMap; - store: StatefulOpenAIResponsesStore; - canonicalInput: OpenAIResponsesInputItem[]; - active: readonly ActiveServerTool[]; - metadata: LatestUpstreamMetadata; - resolveFinalMetadata: (m: EventResultMetadata) => void; -}): AsyncGenerator> { - const { ctx, run, merge, loopState, demoteForcedServerToolChoiceAfterFirstTurn, turn1Iter, dispatchers, store, active, metadata, resolveFinalMetadata } = args; - const baseInput = args.canonicalInput; - let midStreamError: unknown = undefined; - try { - let currentTurn: TurnSummary = yield* turn1Iter; - // Each turn resolves its own cost once its stream has drained, and the - // shim consumes them one at a time, so the running total is complete by - // the time the loop reaches its terminal. - await accumulateBillableUsage(metadata, args.firstResult); - merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); - while (true) { - const turn = currentTurn; - const executedShim = turn.dispatched.length > 0; - - if (turn.terminalStatus.kind === 'failed') { - if (executedShim) yield* materializeServerToolItems(turn.dispatched, merge, store); - yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: turn.terminalStatus.response.error }, active); - return; - } - if (turn.terminalStatus.kind === 'incomplete') { - if (executedShim) yield* materializeServerToolItems(turn.dispatched, merge, store); - yield synthesizeTerminalEnvelope(merge, { kind: 'incomplete', incompleteDetails: turn.terminalStatus.response.incomplete_details }, active); - return; - } - if (turn.terminalStatus.kind === 'bare-error-pre-shell') { - yield synthesizeTerminalEnvelope(merge, { - kind: 'failed', - error: { code: turn.terminalStatus.error.code, message: turn.terminalStatus.error.message }, - }, active); - return; - } - if (!executedShim && !turn.sawClientToolCall) { - yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); - return; - } - - yield* materializeServerToolItems(turn.dispatched, merge, store); - if (turn.sawClientToolCall) { - yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); - return; - } - - // Accumulated output items are fed back as the next turn's input. - // An OpenAI Responses output item is a structural superset of the matching - // input item for every shape we emit here (messages, reasoning, - // function_call / function_call_output, and the server-tool items - // the dispatchers produce), so the reuse is sound; the cast only - // bridges the output/input naming. - const nextCanonicalInput = [ - ...baseInput, - ...materializeAccumulatedOutput(merge).map(item => item as OpenAIResponsesInputItem), - ]; - const nextPayload: CanonicalOpenAIResponsesPayload = { ...ctx.payload, input: transformServerToolItems(nextCanonicalInput, active) }; - if (loopState.remainingToolCalls !== undefined) { - nextPayload.max_tool_calls = Math.max(0, loopState.remainingToolCalls); - } else { - delete nextPayload.max_tool_calls; - } - ctx.payload = nextPayload; - - if (demoteForcedServerToolChoiceAfterFirstTurn) ctx.payload = { ...ctx.payload, tool_choice: 'auto' }; - loopState.iterationCount += 1; - - const nextResult = await run(); - if (nextResult.type !== 'events') { - yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: buildErrorFromResult(nextResult) }, active); - return; - } - metadata.modelIdentity = nextResult.modelIdentity; - metadata.performance = nextResult.performance; - await accumulateBillableUsage(metadata, nextResult); - currentTurn = yield* consumeTurnStreaming(nextResult.events, merge, false, dispatchers, loopState, active); - merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); - } - } catch (error) { - if (merge.lastSeenModel === null) { - midStreamError = error; - throw error; - } - yield synthesizeTerminalEnvelope(merge, { - kind: 'failed', - error: { - code: 'server_error', - message: `Upstream stream failed mid-response: ${error instanceof Error ? error.message : String(error)}`, - }, - }, active); - } finally { - if (midStreamError === undefined) resolveFinalMetadata(metadata); - } -} - -export const withOpenAIResponsesServerToolShim = ( - registrations: readonly ServerToolRegistration[], -): OpenAIResponsesInterceptor => async (ctx, gatewayCtx, run) => { - const active: ActiveServerTool[] = []; - - for (const prepareServerTool of registrations) { - const prepared = await prepareServerTool(ctx, gatewayCtx); - if (prepared.type === 'inactive') continue; - if (prepared.type === 'invalid-request') { - return invalidRequestEnvelope(prepared.message, prepared.param, prepared.code, prepared.errorType); - } - const currentTools = Array.isArray(ctx.payload.tools) ? ctx.payload.tools : []; - const toolName = resolveServerToolName(prepared.baseToolName, currentTools); - const { hosted } = prepared; - let canonicalHostedTool: OpenAIResponsesHostedTool | undefined = undefined; - if (hosted !== undefined) { - const rewrite = rewriteToolsForHostedShim(currentTools, hosted, toolName); - canonicalHostedTool = rewrite.canonicalHostedTool; - ctx.payload = { ...ctx.payload, tools: rewrite.rewritten }; - } - const originalToolChoice = hosted !== undefined - && typeof ctx.payload.tool_choice === 'object' - && ctx.payload.tool_choice !== null - && hosted.hostedTypes.includes(ctx.payload.tool_choice.type) - ? ctx.payload.tool_choice - : undefined; - active.push({ ...prepared, toolName, canonicalHostedTool, originalToolChoice }); - } - - if (active.length === 0) return await run(); - - const rewrittenToolChoice = rewriteHostedToolChoice(ctx.payload.tool_choice, active); - if (rewrittenToolChoice !== ctx.payload.tool_choice) { - ctx.payload = { ...ctx.payload, tool_choice: rewrittenToolChoice }; - } - - const canonicalInput = ctx.payload.input; - const nextInput = transformServerToolItems(canonicalInput, active); - if (nextInput !== canonicalInput) ctx.payload = { ...ctx.payload, input: nextInput }; - - const hostedActive = active.filter( - (entry): entry is ActiveServerTool & { hosted: ServerToolHostedDispatch } => - entry.hosted !== undefined, - ); - if (hostedActive.length === 0) return await run(); - - const dispatchers = new Map(); - for (const entry of hostedActive) dispatchers.set(entry.toolName, entry.hosted.dispatcher); - const loopState: ServerToolLoopState = { - iterationCount: 1, - remainingToolCalls: typeof ctx.payload.max_tool_calls === 'number' ? ctx.payload.max_tool_calls : undefined, - }; - const finalToolChoice = ctx.payload.tool_choice; - const demoteForcedServerToolChoiceAfterFirstTurn = finalToolChoice === 'required' - || (typeof finalToolChoice === 'object' - && finalToolChoice !== null - && finalToolChoice.type === 'function' - && dispatchers.has(finalToolChoice.name)); - - const merge = createMergeState(); - const firstResult = await run(); - if (firstResult.type !== 'events') return firstResult; - const turn1Iter = consumeTurnStreaming(firstResult.events, merge, true, dispatchers, loopState, active); - - let resolveFinalMetadata!: (m: EventResultMetadata) => void; - const shimFinalMetadata = new Promise(resolve => { - resolveFinalMetadata = resolve; - }); - const metadata: LatestUpstreamMetadata = { - modelIdentity: firstResult.modelIdentity, - performance: firstResult.performance, - }; - - return { - ...firstResult, - events: runMultiTurnLoop({ - ctx, - run, - merge, - loopState, - demoteForcedServerToolChoiceAfterFirstTurn, - turn1Iter, - firstResult, - dispatchers, - store: gatewayCtx.store, - canonicalInput, - active, - metadata, - resolveFinalMetadata, - }), - finalMetadata: shimFinalMetadata, - }; -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/stage.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/stage.ts new file mode 100644 index 0000000000..1d68c09909 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/stage.ts @@ -0,0 +1,401 @@ +// The server-tool shim, as a stage. +// +// A hosted tool the upstream does not implement is emulated here: the tool is rewritten into a +// function tool on the way down, the model's call to it is executed by this gateway, its result +// is fed back, and the upstream is asked again — until the model stops calling it. What the +// client sees is one turn, with the hosted tool's own item and lifecycle events in it. +// +// That is a **fork the stage merges**, and it is why this is one stage rather than several. It +// descends more than once, and every descent after the first is asked with the previous turn's +// output folded back into the input; the frames of all of them are spliced into one stream +// under one synthesized response id. Nothing above sees the repetition — which is exactly what +// `writeSettlement` means when it says repetition passes through the stage that observes usage. +// +// Position: directly above the dial, below the fork. So a descent re-dials the *same* candidate, +// and an upstream that refuses mid-loop rides up to `failover`, which re-runs the whole loop +// against the next candidate rather than resuming a half-finished one. +// +// What every descent is billable for travels up separately and is concatenated, not summed: +// each turn is its own upstream call with its own pricing facts — a service tier and an input +// size that a later turn need not share — so the run settles one row per call rather than one +// row whose rate is a guess. + +import { + buildErrorFromRefusal, + consumeTurnStreaming, + createMergeState, + materializeAccumulatedOutput, + materializeServerToolItems, + resolveServerToolName, + rewriteHostedToolChoice, + rewriteToolsForHostedShim, + sumUsage, + synthesizeTerminalEnvelope, + transformServerToolItems, + type ActiveServerTool, + type MergeState, + type ServerToolDispatcher, + type ServerToolHostedDispatch, + type ServerToolLoopState, + type ServerToolRegistration, + type TurnSummary, +} from './shim.ts'; +import type { BillableEntity, Failure } from '../../../pipeline/facts.ts'; +import { isFailure } from '../../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../../pipeline/serve.ts'; +import type { ChatAnswer, ChatFacts } from '../../facts.ts'; +import type { ChatServices } from '../../stages.ts'; +import { defer, defineStage, move, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { + CanonicalOpenAIResponsesPayload, + OpenAIResponsesInputItem, + OpenAIResponsesResult, + OpenAIResponsesStreamEvent, +} from '@floway-dev/protocols/openai-responses'; +import type { ChatTargetApi, ModelCandidate, OpenAIResponsesInvocation } from '@floway-dev/provider'; + +type R = { [P in K]: ChatFacts[P] }; + +/** What comes back through here. The streamed-usage key is the *source* family's rather than + * this protocol's — the same reason a wire is told it — so the slice is open at that one key + * and closed everywhere else. */ +type Answered = R<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'> + & { 'response.chat.openaiResponses.streamedUsage': Deferred | null } + & Record; + +/** One descent, as the loop reads it: the frames it is to splice, and what that call will turn + * out to have been billable for once they run out. */ +interface Turn { + readonly frames: AsyncIterable>; + readonly usage: Deferred | null; +} + +/** What the gateway answers a request its own tool declaration cannot accept. It is this + * gateway's refusal rather than an upstream's, so it carries the envelope a client reads and + * the 400 that goes with it. */ +const invalidRequest = ( + message: string, + param: string | null, + code: string | null | undefined, + errorType = 'invalid_request_error', +): Failure => { + const envelope = { + error: { message, type: errorType, param, code: code === undefined ? 'invalid_request_error' : code }, + }; + return { status: 400, message: JSON.stringify(envelope), body: envelope, envelope }; +}; + +/** + * Prepares every registered tool over the payload this attempt is about to send. + * + * The registrations are handed an invocation rather than facts, because what they read is the + * body and the candidate and nothing else — and the object they read it through is the stage's + * own, built here and never handed anywhere, so shaping it in place shapes nothing shared. + */ +const prepareServerTools = async ( + registrations: readonly ServerToolRegistration[], + invocation: OpenAIResponsesInvocation, + gateway: ChatServices['gateway'], +): Promise<{ readonly active: readonly ActiveServerTool[] } | { readonly refused: Failure }> => { + const active: ActiveServerTool[] = []; + for (const prepareServerTool of registrations) { + const prepared = await prepareServerTool(invocation, gateway); + if (prepared.type === 'inactive') continue; + if (prepared.type === 'invalid-request') { + return { refused: invalidRequest(prepared.message, prepared.param, prepared.code, prepared.errorType) }; + } + const currentTools = Array.isArray(invocation.payload.tools) ? invocation.payload.tools : []; + const toolName = resolveServerToolName(prepared.baseToolName, currentTools); + const { hosted } = prepared; + let canonicalHostedTool; + if (hosted !== undefined) { + const rewrite = rewriteToolsForHostedShim(currentTools, hosted, toolName); + canonicalHostedTool = rewrite.canonicalHostedTool; + invocation.payload = { ...invocation.payload, tools: rewrite.rewritten }; + } + const originalToolChoice = hosted !== undefined + && typeof invocation.payload.tool_choice === 'object' + && invocation.payload.tool_choice !== null + && hosted.hostedTypes.includes(invocation.payload.tool_choice.type) + ? invocation.payload.tool_choice + : undefined; + active.push({ ...prepared, toolName, canonicalHostedTool, originalToolChoice }); + } + return { active }; +}; + +/** What the stage is told rather than reaching for, because reaching for it would be this + * module importing the chain that composes it. `streamedUsage` is the family's own reading key; + * `targetOf` says which wire this candidate is reachable on, which is the one thing a + * registration reads about the dial it is preparing for. */ +export interface ServerToolWiring { + readonly streamedUsage: string; + readonly targetOf: (candidate: ModelCandidate) => ChatTargetApi; +} + +export const runOpenAIResponsesServerTools = ( + registrations: readonly ServerToolRegistration[], + wiring: ServerToolWiring, +) => defineStage< + R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + Answered, + Answered, + Answered, + ChatServices +>({ + name: 'runOpenAIResponsesServerTools', + through: { + request: { + needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { + needs: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers'], + consumes: [], + provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable'], + }, + }, + // A tool declaration this gateway cannot accept is answered here rather than dialled: the + // upstream would have been asked for a tool it does not implement, on a body the shim wrote. + return: { + provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers'], + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // The stage's own view of the turn, which is what a registration reads and shapes. It is + // built here and handed nowhere, so the payload it ends up holding is what descends. + const invocation: OpenAIResponsesInvocation = { + payload: facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload, + candidate, + targetApi: wiring.targetOf(candidate), + headers: new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])), + action: 'generate', + }; + + const prepared = await prepareServerTools(registrations, invocation, use.gateway); + if ('refused' in prepared) { + use.log.debug('refusing a server-tool declaration this gateway cannot accept'); + return move({ + ...facts, + 'response.chat.openaiResponses': prepared.refused, + [wiring.streamedUsage]: null, + 'response.usage.billable': [], + 'response.http.headers': [], + }) as never; + } + const { active } = prepared; + if (active.length === 0) return await next(facts); + + const rewrittenToolChoice = rewriteHostedToolChoice(invocation.payload.tool_choice, active); + if (rewrittenToolChoice !== invocation.payload.tool_choice) { + invocation.payload = { ...invocation.payload, tool_choice: rewrittenToolChoice }; + } + const canonicalInput = invocation.payload.input; + const nextInput = transformServerToolItems(canonicalInput, active); + if (nextInput !== canonicalInput) invocation.payload = { ...invocation.payload, input: nextInput }; + + const hostedActive = active.filter( + (entry): entry is ActiveServerTool & { hosted: ServerToolHostedDispatch } => entry.hosted !== undefined, + ); + // A registration that only rewrote history has nothing to execute, so the turn is an + // ordinary one — dialled with the body it wrote and read by nobody here. + if (hostedActive.length === 0) return await descend(next, facts, invocation.payload); + + const dispatchers = new Map(); + for (const entry of hostedActive) dispatchers.set(entry.toolName, entry.hosted.dispatcher); + const loopState: ServerToolLoopState = { + iterationCount: 1, + remainingToolCalls: typeof invocation.payload.max_tool_calls === 'number' ? invocation.payload.max_tool_calls : undefined, + }; + const finalToolChoice = invocation.payload.tool_choice; + const demoteForcedServerToolChoiceAfterFirstTurn = finalToolChoice === 'required' + || (typeof finalToolChoice === 'object' + && finalToolChoice !== null + && finalToolChoice.type === 'function' + && dispatchers.has(finalToolChoice.name)); + const back = await descend(next, facts, invocation.payload) as Answered & Record; + const first = turnOf(back, wiring.streamedUsage); + // An upstream that refused, or a compaction, is not a turn the loop can splice: it rides up + // as it came, and the tools this turn declared were simply never called. + if (first === null) return back as never; + + const merge = createMergeState(); + const billed: BillableEntity[] = []; + let settle!: (outcome: StreamOutcome) => void; + // Every descent's reading is awaited inside the loop, so this is complete by the time the + // spliced stream ends — which is the moment the run is answerable for what it billed. + const usage = defer(new Promise(resolve => { settle = resolve; })); + + return { + ...back, + 'response.chat.openaiResponses': move({ + kind: 'stream' as const, + frames: { + [Symbol.asyncIterator]: () => spliceTurns({ + next, + facts, + invocation, + merge, + loopState, + demoteForcedServerToolChoiceAfterFirstTurn, + first, + dispatchers, + store: use.gateway.store, + canonicalInput, + active, + billed, + settle, + streamedUsage: wiring.streamedUsage, + }), + }, + }), + [wiring.streamedUsage]: move(usage), + // What was known when this stage handed up, which is what the first call reported. The + // figure every call turned out to cost arrives with the stream, through the reading above. + 'response.usage.billable': back['response.usage.billable'], + } as never; + }, +}); + +/** One descent, asked with the payload the shim wrote. */ +const descend = async ( + next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>) => Promise, + facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + payload: CanonicalOpenAIResponsesPayload, +): Promise => await next(move({ ...facts, 'request.chat.openaiResponses': move(payload) })); + +/** The descent's answer as a turn, or `null` where it was not one the loop can splice. The + * reading is read at the key the wire was told, which is the source family's own. */ +const turnOf = (back: Answered, streamedUsage: string): Turn | null => { + const answer = back['response.chat.openaiResponses'] as ChatAnswer; + if (isFailure(answer) || answer.kind !== 'stream') return null; + return { + frames: answer.frames as AsyncIterable>, + usage: back[streamedUsage] as Deferred | null, + }; +}; + +/** What a turn cost, once its own frames have run out. The loop reads them one at a time, so + * the running total is complete when the spliced stream reaches its terminal. */ +const accumulate = async (billed: BillableEntity[], turn: Turn): Promise => { + if (turn.usage === null) return false; + const outcome = await turn.usage; + billed.push(...outcome.billable); + return outcome.failed; +}; + +async function* spliceTurns(args: { + next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>) => Promise; + facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>; + invocation: OpenAIResponsesInvocation; + merge: MergeState; + loopState: ServerToolLoopState; + demoteForcedServerToolChoiceAfterFirstTurn: boolean; + first: Turn; + dispatchers: ReadonlyMap; + store: ChatServices['gateway']['store']; + canonicalInput: OpenAIResponsesInputItem[]; + active: readonly ActiveServerTool[]; + billed: BillableEntity[]; + settle: (outcome: StreamOutcome) => void; + streamedUsage: string; +}): AsyncGenerator> { + const { invocation, merge, loopState, demoteForcedServerToolChoiceAfterFirstTurn, dispatchers, store, active, billed, settle } = args; + const baseInput = args.canonicalInput; + let failed = false; + let midStreamError: unknown = undefined; + try { + let currentTurn: TurnSummary = yield* consumeTurnStreaming(args.first.frames, merge, true, dispatchers, loopState, active); + failed = await accumulate(billed, args.first) || failed; + merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); + for (;;) { + const turn = currentTurn; + const executedShim = turn.dispatched.length > 0; + + if (turn.terminalStatus.kind === 'failed') { + if (executedShim) yield* materializeServerToolItems(turn.dispatched, merge, store); + yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: turn.terminalStatus.response.error }, active); + return; + } + if (turn.terminalStatus.kind === 'incomplete') { + if (executedShim) yield* materializeServerToolItems(turn.dispatched, merge, store); + yield synthesizeTerminalEnvelope(merge, { kind: 'incomplete', incompleteDetails: turn.terminalStatus.response.incomplete_details }, active); + return; + } + if (turn.terminalStatus.kind === 'bare-error-pre-shell') { + yield synthesizeTerminalEnvelope(merge, { + kind: 'failed', + error: { code: turn.terminalStatus.error.code, message: turn.terminalStatus.error.message }, + }, active); + return; + } + if (!executedShim && !turn.sawClientToolCall) { + yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); + return; + } + + yield* materializeServerToolItems(turn.dispatched, merge, store); + if (turn.sawClientToolCall) { + yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); + return; + } + + // Accumulated output items are fed back as the next turn's input. An OpenAI Responses + // output item is a structural superset of the matching input item for every shape emitted + // here — messages, reasoning, function_call / function_call_output, and the server-tool + // items the dispatchers produce — so the reuse is sound; the cast only bridges the + // output/input naming. + const nextCanonicalInput = [ + ...baseInput, + ...materializeAccumulatedOutput(merge).map(item => item as OpenAIResponsesInputItem), + ]; + const { max_tool_calls: _spent, ...withoutCap } = invocation.payload; + invocation.payload = { + ...withoutCap, + input: transformServerToolItems(nextCanonicalInput, active), + ...(loopState.remainingToolCalls === undefined ? {} : { max_tool_calls: Math.max(0, loopState.remainingToolCalls) }), + ...(demoteForcedServerToolChoiceAfterFirstTurn ? { tool_choice: 'auto' as const } : {}), + }; + loopState.iterationCount += 1; + + const back = await descend(args.next, args.facts, invocation.payload); + const nextTurn = turnOf(back, args.streamedUsage); + if (nextTurn === null) { + yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: errorOf(back) }, active); + return; + } + failed = await accumulate(billed, nextTurn) || failed; + currentTurn = yield* consumeTurnStreaming(nextTurn.frames, merge, false, dispatchers, loopState, active); + merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); + } + } catch (error) { + if (merge.lastSeenModel === null) { + midStreamError = error; + throw error; + } + failed = true; + yield synthesizeTerminalEnvelope(merge, { + kind: 'failed', + error: { + code: 'server_error', + message: `Upstream stream failed mid-response: ${error instanceof Error ? error.message : String(error)}`, + }, + }, active); + } finally { + // A run that threw before a model was known has no answer to settle from; every other way + // out of the loop leaves what each call billed, and whether the turn got where it said. + if (midStreamError === undefined) settle({ billable: billed, failed }); + } +} + +/** A descent that did not answer with a stream, as the error the synthesized envelope carries. + * A compaction reaches here only if a client asked one to run tools, which no client does. */ +const errorOf = (back: Answered): NonNullable => { + const answer = back['response.chat.openaiResponses'] as ChatAnswer; + return isFailure(answer) + ? buildErrorFromRefusal(answer.status, answer.message) + : { code: 'server_error', message: 'the upstream answered a server-tool turn with a compaction' }; +}; From b2d17cbdc0f122ec8d56b12159a03f48460aa71f Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 09:41:48 +0800 Subject: [PATCH 090/156] feat(anthropic-messages): bring the web-search server tool back, as a stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic exposes `web_search_*` as native server tools. An upstream that cannot serve them — every non-Anthropic-Messages target, and a native one whose operator says so — was being sent the native declaration anyway: the shim had never become a stage, so it was reachable only from its own tests. `:countTokens` had already been ported and read the `messages-web-search-shim` flag, which made the gap visible from one side of the same operation. One descent, unlike the OpenAI Responses shim: Anthropic carries the tool result inside the same turn, so there is nothing to ask again for. The stage is a request rewrite and a response rewrite around one dial, below the Claude Code probe — a probe turn declares no tools — and above the dial whose body it writes. A search backend the operator never configured now raises rather than returning an internal-error result. A turn nobody can answer is this gateway's own fault, and the seam turns a raise into the same envelope with the sentence and the stack on it — which is more than the result shape carried. Its 34 tests keep every assertion. Three drove the interceptor and now drive the stage through a scripted ending; two drove the count-tokens half, which is a stage of its own already, and now drive the preparation both halves share. With that the last `interceptors/` directory under `chat/` is gone: the probe's predicate, the two OpenAI Responses tools and this one all sit beside the stages that compose them. --- .../anthropic-messages/web-search-drive.ts | 79 +++++++++++ .../web-search-shim_test.ts | 98 +++++++------- .../image-generation-integration_test.ts | 3 +- .../server-tools/image-generation_test.ts | 2 +- .../server-tools/shim_test.ts | 11 +- .../chat/anthropic-messages/count-tokens.ts | 2 +- .../anthropic-messages/interceptors/types.ts | 23 ---- .../chat/anthropic-messages/pipeline.ts | 4 + .../{interceptors => }/web-search-shim.ts | 80 ++++-------- .../anthropic-messages/web-search-tool.ts | 123 ++++++++++++++++++ .../openai-responses/interceptors/types.ts | 13 -- 11 files changed, 284 insertions(+), 154 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts rename packages/gateway/__tests__/data-plane/chat/anthropic-messages/{interceptors => }/web-search-shim_test.ts (94%) delete mode 100644 packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts rename packages/gateway/src/data-plane/chat/anthropic-messages/{interceptors => }/web-search-shim.ts (91%) create mode 100644 packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts delete mode 100644 packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts new file mode 100644 index 0000000000..302b05de2b --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts @@ -0,0 +1,79 @@ +// Driving Anthropic's web-search stage from a test that scripts the dial. +// +// What a test scripts is the *ending* — the one answer the stage descends for. Everything above +// it is the real chain: the preparation rewrites the body, the stage descends, and the frames +// come back rewritten into Anthropic's native shape. +// +// `invocation.payload` is written back on the descent, so a test that asserts on what the +// upstream was asked for reads the same field it always did. + +import { runAnthropicMessagesWebSearchTool } from '../../../../src/data-plane/chat/anthropic-messages/web-search-tool.ts'; +import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { AnthropicMessagesInvocation, ExecuteResult, ModelCandidate } from '@floway-dev/provider'; + +const ANSWER = 'response.chat.anthropicMessages'; + +type Result = ExecuteResult>; + +/** A scripted refusal as the failure value a chain carries. */ +const failureOf = (result: Exclude) => ({ + status: result.type === 'internal-error' ? result.status : result.status, + message: result.type === 'internal-error' ? result.error.message : new TextDecoder().decode(result.body), +}); + +/** What the run answered with, in the shape the assertions read. */ +const resultOf = (facts: Record): Result => { + const answer = facts[ANSWER] as { status?: number; message?: string; body?: unknown; kind?: string; frames?: unknown }; + if (answer.kind === 'stream') { + return { type: 'events', events: answer.frames as AsyncIterable>, modelIdentity: undefined } as unknown as Result; + } + return { + type: 'api-error', + source: 'gateway', + status: answer.status!, + headers: new Headers({ 'content-type': 'application/json' }), + body: new TextEncoder().encode(answer.message!), + } as Result; +}; + +export const driveWebSearchStage = async ( + invocation: AnthropicMessagesInvocation, + gatewayCtx: ChatGatewayCtx, + dial: () => Promise, +): Promise => { + const ending = defineStage, Record>({ + name: 'scriptedDial', + return: { provides: [ANSWER, 'response.usage.billable', 'response.http.headers'] }, + execute: async facts => { + invocation.payload = facts['request.chat.anthropicMessages'] as AnthropicMessagesPayload; + const result = await dial(); + return move({ + ...facts, + [ANSWER]: result.type === 'events' + ? { kind: 'stream' as const, frames: result.events } + : failureOf(result), + 'response.usage.billable': [], + 'response.http.headers': [], + }); + }, + }); + + const chain = compose, Record>('webSearchToolUnderTest', [ + runAnthropicMessagesWebSearchTool({ targetOf: () => invocation.targetApi }) as never, + ending, + ]); + + const { facts } = await run(chain, move({ + 'request.chat.anthropicMessages': invocation.payload, + 'route.attempt': { upstreamId: 'up_test', modelId: invocation.payload.model, flags: [] }, + }), { + gateway: gatewayCtx, + background: () => {}, + resolveAttempt: (): ModelCandidate => invocation.candidate, + } as never); + + return resultOf(facts); +}; diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/web-search-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts similarity index 94% rename from packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/web-search-shim_test.ts rename to packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts index 83c1d33a32..81805d0d80 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/web-search-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import type { AnthropicMessagesInvocation } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/types.ts'; +import { driveWebSearchStage } from './web-search-drive.ts'; import { decodeWebSearchCitationPayload, decodeWebSearchResultPayload, @@ -9,14 +9,12 @@ import { type AnthropicMessagesWebSearchShimState, prepareAnthropicMessagesWebSearchShimRequest, rewriteAnthropicMessagesWebSearchEventsToNative, - withAnthropicMessagesWebSearchRequestPrepared, - withAnthropicMessagesWebSearchShim, -} from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts'; -import { DEFAULT_WEB_SEARCH_CONFIG } from '../../../../../src/data-plane/tools/web-search/config.ts'; -import type { WebSearchProvider, WebSearchProviderResult } from '../../../../../src/data-plane/tools/web-search/types.ts'; -import { initRepo } from '../../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +} from '../../../../src/data-plane/chat/anthropic-messages/web-search-shim.ts'; +import { DEFAULT_WEB_SEARCH_CONFIG } from '../../../../src/data-plane/tools/web-search/config.ts'; +import type { WebSearchProvider, WebSearchProviderResult } from '../../../../src/data-plane/tools/web-search/types.ts'; +import { initRepo } from '../../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../../repo/memory.ts'; +import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; import { anthropicMessagesProtocolFrameToSSEFrame } from '@floway-dev/protocols/anthropic-messages'; import type { AnthropicMessagesAssistantContentBlock, @@ -30,6 +28,7 @@ import type { AnthropicMessagesUserContentBlock, } from '@floway-dev/protocols/anthropic-messages'; import { type ProtocolFrame, eventFrame } from '@floway-dev/protocols/common'; +import type { AnthropicMessagesInvocation } from '@floway-dev/provider'; import { assertEquals, assertExists, assertRejects, stubModelCandidate } from '@floway-dev/test-utils'; const testTelemetryModelIdentity = { @@ -587,44 +586,34 @@ test('generation and count_tokens prepare identical web-search request payloads' const generationInvocation = invocation(structuredClone(source)); const countInvocation = invocation(structuredClone(source)); - await withAnthropicMessagesWebSearchShim(generationInvocation, mockChatGatewayCtx(), () => Promise.resolve({ + await driveWebSearchStage(generationInvocation, mockChatGatewayCtx(), () => Promise.resolve({ type: 'events', events: toAsyncIterable>([]), modelIdentity: testTelemetryModelIdentity, })); - const countResponse = await withAnthropicMessagesWebSearchRequestPrepared( - countInvocation, - mockChatGatewayCtx(), - () => Promise.resolve(new Response(null, { status: 204 })), - ); - - assertEquals(countResponse.status, 204); - assertEquals(countInvocation.payload, generationInvocation.payload); + // `:countTokens` has no stream to rewrite, so its stage prepares the same request and dials + // it — measuring what generation would have sent rather than what the client wrote. + const counted = prepareAnthropicMessagesWebSearchShimRequest(countInvocation.payload); + assertEquals(counted.type, 'ok'); + assertEquals(counted.type === 'ok' ? counted.payload : undefined, generationInvocation.payload); const rewrittenTool = generationInvocation.payload.tools?.[0] as AnthropicMessagesClientTool; assertEquals(rewrittenTool.name, 'web_search'); assertEquals('type' in rewrittenTool, false); }); test('count_tokens returns a native error response for invalid web-search tools', async () => { - const response = await withAnthropicMessagesWebSearchRequestPrepared( - invocation({ - model: 'claude-test', - max_tokens: 64, - messages: [{ role: 'user', content: 'search' }], - tools: [{ type: 'web_search_20260209' }, { type: 'web_search_20260209' }], - }), - mockChatGatewayCtx(), - () => Promise.reject(new Error('run should not be called')), - ); - - assertEquals(response.status, 400); - assertEquals(await response.json(), { - type: 'error', - error: { - type: 'invalid_request_error', - message: 'Only one native web search tool definition is supported per request.', - }, + const prepared = prepareAnthropicMessagesWebSearchShimRequest({ + model: 'claude-test', + max_tokens: 64, + messages: [{ role: 'user', content: 'search' }], + tools: [{ type: 'web_search_20260209' }, { type: 'web_search_20260209' }], }); + + assertEquals(prepared.type, 'invalid-request'); + assertEquals( + prepared.type === 'invalid-request' ? prepared.message : undefined, + 'Only one native web search tool definition is supported per request.', + ); }); const runReplayOnlyShim = async (messageId: string): Promise[]> => { @@ -632,7 +621,7 @@ const runReplayOnlyShim = async (messageId: string): Promise + const result = await driveWebSearchStage(invocation(payload), mockChatGatewayCtx(), () => Promise.resolve({ type: 'events', events: toAsyncIterable( @@ -672,23 +661,30 @@ const runReplayOnlyShim = async (messageId: string): Promise { +// A turn that needs a search backend the operator never configured is unanswerable, and that is +// this gateway's own fault rather than a refusal to fail over — so it raises, and the seam turns +// it into the internal-error envelope with the sentence and the stack on it. +test('the web-search stage raises when the request needs a search provider the operator disabled', async () => { await initDisabledSearchRepo(); - const result = await withAnthropicMessagesWebSearchShim( - invocation({ - model: 'claude-test', - max_tokens: 64, - messages: [{ role: 'user', content: 'latest React docs' }], - tools: [{ type: 'web_search_20260209' }], - }), mockChatGatewayCtx(), - () => Promise.reject(new Error('run should not be called')), - ); - - assertEquals(result.type, 'internal-error'); + let raised: unknown; + try { + await driveWebSearchStage( + invocation({ + model: 'claude-test', + max_tokens: 64, + messages: [{ role: 'user', content: 'latest React docs' }], + tools: [{ type: 'web_search_20260209' }], + }), mockChatGatewayCtx(), + () => Promise.reject(new Error('run should not be called')), + ); + } catch (error) { raised = error; } + + assertEquals(raised instanceof Error, true); + assertEquals((raised as Error).message, 'Native Anthropic Messages web search requires an enabled search provider.'); }); -test('withAnthropicMessagesWebSearchShim allows replay-only history when the search provider is disabled', async () => { +test('the web-search stage allows replay-only history when the search provider is disabled', async () => { const collected = await runReplayOnlyShim('msg_replay_only'); const events = collected.flatMap(frame => (frame.type === 'event' ? [frame.event] : [])); @@ -696,7 +692,7 @@ test('withAnthropicMessagesWebSearchShim allows replay-only history when the sea assertEquals(citationsDelta?.delta.type === 'citations_delta' ? citationsDelta.delta.citation.type : undefined, 'web_search_result_location'); }); -test('withAnthropicMessagesWebSearchShim emits native-like citation deltas for replay-only history', async () => { +test('the web-search stage emits native-like citation deltas for replay-only history', async () => { const collected = await runReplayOnlyShim('msg_replay_only_stream'); const frames = collected.map(anthropicMessagesProtocolFrameToSSEFrame).filter(frame => frame !== null); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts index 8017b2460c..cc25d93061 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts @@ -1,7 +1,6 @@ import { beforeEach, test, vi } from 'vitest'; import { driveServerToolStage } from './drive.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; import { initRepo } from '../../../../../src/repo/index.ts'; import { InMemoryRepo } from '../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; @@ -9,7 +8,7 @@ import { createInMemoryImageProcessor, initExternalResourceFetcher, initImagePro import { eventFrame } from '@floway-dev/protocols/common'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesResult, OpenAIResponsesStreamEvent, OpenAIResponsesTool, OpenAIResponsesToolChoice } from '@floway-dev/protocols/openai-responses'; -import { type EventResult, type ExecuteResult, type FlagId, type OpenAIImagesEditsRequest } from '@floway-dev/provider'; +import { type EventResult, type ExecuteResult, type FlagId, type OpenAIImagesEditsRequest, type OpenAIResponsesInvocation } from '@floway-dev/provider'; import { assert, assertEquals, assertStringIncludes, stubModelCandidate } from '@floway-dev/test-utils'; // Dirty integration harness: mock the model registry so the image backend is a diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation_test.ts index 2311d5ffe8..60529b98e2 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation_test.ts @@ -1,6 +1,5 @@ import { beforeEach, test } from 'vitest'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; import { buildImageGenerationFunctionTool, createImageSourceInspector, @@ -24,6 +23,7 @@ import { InMemoryRepo } from '../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; import { initExternalResourceFetcher } from '@floway-dev/platform'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputImage, OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesTool } from '@floway-dev/protocols/openai-responses'; +import type { OpenAIResponsesInvocation } from '@floway-dev/provider'; import { assert, assertEquals, assertFalse, assertStringIncludes, assertThrows, stubModelCandidate } from '@floway-dev/test-utils'; const PNG_B64 = 'aGVsbG8='; // "hello" — any decodable base64 works for source tests. diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts index d1f61b1a27..333eba38d9 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts @@ -1,7 +1,6 @@ import { beforeEach, test, vi } from 'vitest'; import { driveServerToolStage } from './drive.ts'; -import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; import { createNonOpenAIResponsesSourceStore } from '../../../../../src/data-plane/chat/openai-responses/items/store.ts'; import { consumeTurnStreaming, @@ -46,7 +45,7 @@ import type { OpenAIResponsesToolChoice, OpenAIResponsesWebSearchAction, } from '@floway-dev/protocols/openai-responses'; -import { type EventResult, type ExecuteResult, type FlagId } from '@floway-dev/provider'; +import { type EventResult, type ExecuteResult, type FlagId, type OpenAIResponsesInvocation } from '@floway-dev/provider'; import { assert, assertEquals, assertFalse, stubModelCandidate } from '@floway-dev/test-utils'; const withOpenAIResponsesWebSearchShim = driveServerToolStage([webSearchServerTool]); @@ -354,11 +353,11 @@ const collectFrames = async (iter: AsyncIterable): Promise => { return out; }; -// The shim streams lazily: backend calls, ctx.payload.input mutation, and -// subsequent run() calls all happen inside the events generator. Tests -// asserting on side-effects MUST drain the events stream first. +// The stage streams lazily: backend calls, the payload each descent sends, and the descents +// themselves all happen inside the events generator. Tests asserting on side-effects MUST drain +// the events stream first. const runShimAndDrain = async ( - shim: OpenAIResponsesInterceptor, + shim: ReturnType, inv: OpenAIResponsesInvocation, gatewayCtx: ChatGatewayCtx, run: () => Promise>>, diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index 3c7ad0dda2..422c36015a 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -29,8 +29,8 @@ import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; import { renderAnthropicMessagesError } from './errors.ts'; -import { prepareAnthropicMessagesWebSearchShimRequest } from './interceptors/web-search-shim.ts'; import type { AnthropicMessagesFacts } from './pipeline.ts'; +import { prepareAnthropicMessagesWebSearchShimRequest } from './web-search-shim.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts deleted file mode 100644 index 418b6faadf..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult, AnthropicMessagesInvocation } from '@floway-dev/provider'; - -export type { AnthropicMessagesInvocation }; - -export type AnthropicMessagesInterceptor = Interceptor< - AnthropicMessagesInvocation, - GatewayCtx, - ExecuteResult> ->; - -// count_tokens is a one-shot, non-streaming HTTP exchange whose terminal -// returns the raw upstream `Response`. Shared entries must therefore be pure -// header/payload mutators; post-run stream inspection is not portable to this -// result type. -export type AnthropicMessagesCountTokensInterceptor = Interceptor< - AnthropicMessagesInvocation, - GatewayCtx, - Response ->; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 7fd68af931..88d00ff0d9 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -25,6 +25,7 @@ import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; import { isClaudeCodeProbe, probeFrames } from './claude-code-probe.ts'; import { renderAnthropicMessagesError } from './errors.ts'; import { createAnthropicMessagesBillableUsageReader } from './usage.ts'; +import { runAnthropicMessagesWebSearchTool } from './web-search-tool.ts'; import { recordStream, streamReferenceOf } from '../../../dump/turn-dump.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; @@ -519,6 +520,9 @@ export const anthropicMessagesServePipeline = (payload: AnthropicMessagesPayload }), materializeAttempt('request.chat.anthropicMessages'), answerClaudeCodeProbe, + // Below the probe, because a probe turn declares no tools; above the dial, because what it + // rewrites is the body that dial sends and the frames that dial hands back. + runAnthropicMessagesWebSearchTool({ targetOf: candidate => anthropicMessagesTarget.pick(candidate.model.endpoints) }), dialChatWire({ source: 'request.chat.anthropicMessages', needs: ['request.chat.anthropicMessages', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts similarity index 91% rename from packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts rename to packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts index 0facbf88e6..e90269a1d5 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts @@ -1,11 +1,10 @@ -import type { AnthropicMessagesCountTokensInterceptor, AnthropicMessagesInterceptor, AnthropicMessagesInvocation } from './types.ts'; -import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../shared/base64url-json.ts'; -import { isJsonObject } from '../../../../shared/json-helpers.ts'; -import { loadWebSearchConfig } from '../../../tools/web-search/config.ts'; -import { resolveConfiguredWebSearchProvider } from '../../../tools/web-search/provider.ts'; -import { runWebSearchAndRecordUsage } from '../../../tools/web-search/search.ts'; -import type { WebSearchProvider, WebSearchProviderName, WebSearchProviderRequest, WebSearchProviderResult } from '../../../tools/web-search/types.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../shared/base64url-json.ts'; +import { isJsonObject } from '../../../shared/json-helpers.ts'; +import { loadWebSearchConfig } from '../../tools/web-search/config.ts'; +import { resolveConfiguredWebSearchProvider } from '../../tools/web-search/provider.ts'; +import { runWebSearchAndRecordUsage } from '../../tools/web-search/search.ts'; +import type { WebSearchProvider, WebSearchProviderName, WebSearchProviderRequest, WebSearchProviderResult } from '../../tools/web-search/types.ts'; import type { AnthropicMessagesAssistantContentBlock, AnthropicMessagesAssistantInputContentBlock, @@ -25,7 +24,8 @@ import type { } from '@floway-dev/protocols/anthropic-messages'; import { ANTHROPIC_MESSAGES_WEB_SEARCH_ERROR_CODES } from '@floway-dev/protocols/anthropic-messages'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { providerModelOf, internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; +import type { AnthropicMessagesInvocation } from '@floway-dev/provider'; +import { providerModelOf } from '@floway-dev/provider'; const MAX_QUERY_LENGTH = 1000; const WEB_SEARCH_TOOL_NAME = 'web_search'; @@ -847,7 +847,7 @@ const anthropicMessagesWebSearchInvalidRequestBody = (message: string) => ({ }, }); -const buildSyntheticInvalidRequestUpstreamError = (message: string) => ({ +export const buildSyntheticInvalidRequestUpstreamError = (message: string) => ({ type: 'api-error' as const, source: 'gateway' as const, status: 400, @@ -855,33 +855,23 @@ const buildSyntheticInvalidRequestUpstreamError = (message: string) => ({ body: new TextEncoder().encode(JSON.stringify(anthropicMessagesWebSearchInvalidRequestBody(message))), }); -const buildInvalidRequestResponse = (message: string): Response => - Response.json(anthropicMessagesWebSearchInvalidRequestBody(message), { status: 400 }); - -const resolveActiveAnthropicMessagesWebSearchProvider = async (apiKeyId: string): Promise<{ type: 'ok'; provider: ActiveAnthropicMessagesWebSearchProvider } | ReturnType> => { +/** + * The backend a search that will actually run needs. + * + * An operator who configured none has made this turn unanswerable, and that is this gateway's + * own fault rather than a refusal to fail over — so it raises, with the sentence naming which + * of the two states the configuration is in. + */ +export const resolveActiveAnthropicMessagesWebSearchProvider = async (apiKeyId: string): Promise => { const webSearchConfig = await loadWebSearchConfig(); const configuredProvider = resolveConfiguredWebSearchProvider(webSearchConfig); - if (configuredProvider.type === 'enabled') { - return { - type: 'ok', - provider: { - providerName: configuredProvider.provider, - impl: configuredProvider.impl, - apiKeyId, - }, - }; + return { providerName: configuredProvider.provider, impl: configuredProvider.impl, apiKeyId }; } - - return internalErrorResult( - 500, - toInternalDebugError( - new Error( - configuredProvider.type === 'disabled' - ? 'Native Anthropic Messages web search requires an enabled search provider.' - : `Native Anthropic Messages web search is missing the configured ${configuredProvider.provider} credential.`, - ), - ), + throw new Error( + configuredProvider.type === 'disabled' + ? 'Native Anthropic Messages web search requires an enabled search provider.' + : `Native Anthropic Messages web search is missing the configured ${configuredProvider.provider} credential.`, ); }; @@ -892,7 +882,7 @@ type PrepareAnthropicMessagesWebSearchInvocationResult = | { type: 'invalid-request'; message: string } | { type: 'prepared'; state: PreparedAnthropicMessagesWebSearchShimState }; -const prepareAnthropicMessagesWebSearchInvocation = (ctx: AnthropicMessagesInvocation): PrepareAnthropicMessagesWebSearchInvocationResult => { +export const prepareAnthropicMessagesWebSearchInvocation = (ctx: AnthropicMessagesInvocation): PrepareAnthropicMessagesWebSearchInvocationResult => { if (ctx.targetApi === 'anthropicMessages' && !providerModelOf(ctx.candidate).enabledFlags.has('messages-web-search-shim')) { return { type: 'inactive' }; } @@ -918,27 +908,3 @@ const prepareAnthropicMessagesWebSearchInvocation = (ctx: AnthropicMessagesInvoc * `messages-web-search-shim` flag for native Anthropic Messages targets (the upstream * may or may not be able to serve web_search natively). */ -export const withAnthropicMessagesWebSearchShim: AnthropicMessagesInterceptor = async (ctx, gatewayCtx, run) => { - const prepared = prepareAnthropicMessagesWebSearchInvocation(ctx); - if (prepared.type === 'inactive') return await run(); - if (prepared.type === 'invalid-request') return buildSyntheticInvalidRequestUpstreamError(prepared.message); - - const provider = prepared.state.mode === 'active' ? await resolveActiveAnthropicMessagesWebSearchProvider(gatewayCtx.apiKeyId) : { type: 'ok' as const, provider: undefined }; - if (provider.type !== 'ok') return provider; - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: rewriteAnthropicMessagesWebSearchEventsToNative(result.events, prepared.state, provider.provider), - }; -}; - -// count_tokens has no stream to rewrite, but it must measure the same client- -// tool and replay-history request shape that generation sends upstream. -export const withAnthropicMessagesWebSearchRequestPrepared: AnthropicMessagesCountTokensInterceptor = async (ctx, _gatewayCtx, run) => { - const prepared = prepareAnthropicMessagesWebSearchInvocation(ctx); - if (prepared.type === 'invalid-request') return buildInvalidRequestResponse(prepared.message); - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts new file mode 100644 index 0000000000..6ef71f68d8 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts @@ -0,0 +1,123 @@ +// Anthropic's web-search server tool, as a stage. +// +// Anthropic exposes `web_search_*` as native server tools. An upstream that cannot serve them — +// every non-Anthropic-Messages target, and a native one whose operator says so — gets the tool +// rewritten into an ordinary client tool on the way down, each search the model issues executed +// by this gateway, and the answer rewritten back into Anthropic's own +// `server_tool_use` / `web_search_tool_result` shape on the way up. +// +// One descent, unlike the OpenAI Responses shim: Anthropic's protocol carries the tool result +// inside the same turn, so there is nothing to ask again for. What this stage is, then, is a +// request rewrite and a response rewrite around one dial. + +import { + buildSyntheticInvalidRequestUpstreamError, + prepareAnthropicMessagesWebSearchInvocation, + resolveActiveAnthropicMessagesWebSearchProvider, + rewriteAnthropicMessagesWebSearchEventsToNative, +} from './web-search-shim.ts'; +import type { Failure } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import type { ChatAnswer, ChatFacts } from '../facts.ts'; +import type { ChatServices } from '../stages.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; + +type M = { [P in K]: ChatFacts[P] }; + +type Answered = M<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers'> & Record; + +/** The gateway's own refusal, in Anthropic's words. It is written here rather than dialled, + * because a tool declaration this shim cannot execute would reach the upstream as a body this + * gateway wrote. */ +const refusal = (message: string): Failure => { + const built = buildSyntheticInvalidRequestUpstreamError(message); + const text = new TextDecoder().decode(built.body); + return { status: built.status, message: text, body: JSON.parse(text) as unknown }; +}; + +/** What the stage is told rather than reaching for: which wire this candidate is reachable on, + * which is what decides whether the upstream can carry Anthropic's server tools at all. */ +export interface WebSearchWiring { + readonly targetOf: (candidate: ModelCandidate) => ChatTargetApi; +} + +export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => defineStage< + M<'request.chat.anthropicMessages' | 'route.attempt'>, + M<'request.chat.anthropicMessages' | 'route.attempt'>, + Answered, + Answered, + Answered, + ChatServices +>({ + name: 'runAnthropicMessagesWebSearchTool', + through: { + request: { + needs: ['request.chat.anthropicMessages', 'route.attempt'], + consumes: [], + provides: ['request.chat.anthropicMessages'], + }, + response: { + needs: ['response.chat.anthropicMessages'], + consumes: [], + provides: ['response.chat.anthropicMessages'], + }, + }, + // A declaration this gateway cannot execute, and a search backend the operator has not + // configured, are both answered here: neither is a body an upstream should be asked about. + return: { + provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers'], + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // The stage's own view of the turn, which is what the preparation reads and shapes. It is + // built here and handed nowhere, so the payload it ends up holding is what descends. + const invocation = { + payload: facts['request.chat.anthropicMessages'], + candidate, + targetApi: wiring.targetOf(candidate), + headers: new Headers(), + }; + + const prepared = prepareAnthropicMessagesWebSearchInvocation(invocation); + if (prepared.type === 'inactive') return await next(facts); + if (prepared.type === 'invalid-request') { + use.log.debug('refusing a web-search declaration this gateway cannot execute'); + return move({ + ...facts, + 'response.chat.anthropicMessages': refusal(prepared.message), + 'response.usage.billable': [], + 'response.http.headers': [], + }) as never; + } + + // A search that will actually run needs a backend; one the operator never configured + // raises, because a turn nobody can answer is this gateway's own fault to surface. + const backend = prepared.state.mode === 'active' + ? await resolveActiveAnthropicMessagesWebSearchProvider(use.gateway.apiKeyId) + : undefined; + + const back = await next(move({ + ...facts, + 'request.chat.anthropicMessages': move(invocation.payload as AnthropicMessagesPayload), + })) as Answered; + + const answer = back['response.chat.anthropicMessages'] as ChatAnswer; + if (isFailure(answer) || answer.kind !== 'stream') return back as never; + return { + ...back, + 'response.chat.anthropicMessages': move({ + kind: 'stream' as const, + frames: { + [Symbol.asyncIterator]: () => rewriteAnthropicMessagesWebSearchEventsToNative( + answer.frames as AsyncIterable>, + prepared.state, + backend, + ), + }, + }), + } as never; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts deleted file mode 100644 index 300c417546..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult, OpenAIResponsesInvocation } from '@floway-dev/provider'; - -export type { OpenAIResponsesInvocation }; - -export type OpenAIResponsesInterceptor = Interceptor< - OpenAIResponsesInvocation, - ChatGatewayCtx, - ExecuteResult> ->; From abcd199fefa5e42bf4db37ae12a8b1dccac4b0d4 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 09:55:00 +0800 Subject: [PATCH 091/156] refactor(chat): erase the surface the pipeline replaced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `chat/interceptors.ts` was the last file named for the concept the chain replaced. What it holds is what every family's prose already calls them — the rules a chain composes — so it is `chat/rules.ts`, and its header states what a rule declares rather than what it used to do procedurally. The prose that survived the port is corrected rather than reworded. Three comments claimed a shim "never became a stage and the interceptor array that ran it is gone"; both shims are stages now, so what they say instead is where each one sits and what it does there. Two more described a rule standing down on a target-API guard, where position is what says it — a stage below the fork runs only on the wire the fork chose. One of those claims turned out to be a real gap rather than only stale prose. The compaction chain composed no server-tool stage, so a history carrying an `image_generation_call` or `web_search_call` echoed from an earlier turn reached the upstream unrewritten. The stage is in that chain now, for one of its two halves: a summarization declares no hosted tool, so nothing dispatches and the loop never runs — which is what keeps a compaction from calling out on the caller's behalf — while the history rewrite does. --- ...nterceptors_test.ts => chat-rules_test.ts} | 2 +- .../openai-chat-completions-pipeline_test.ts | 6 +-- .../openai-responses/server-tools/drive.ts | 2 +- .../routes-openai-chat-completions_test.ts | 2 +- .../data-plane/chat/translated-wires_test.ts | 4 +- .../anthropic-messages/claude-code-probe.ts | 2 +- .../chat/anthropic-messages/count-tokens.ts | 2 +- .../chat/anthropic-messages/pipeline.ts | 19 ++++----- .../gemini-generate-content/count-tokens.ts | 2 +- .../chat/gemini-generate-content/pipeline.ts | 15 +++---- .../chat/openai-chat-completions/pipeline.ts | 8 ++-- .../chat/openai-responses/compact.ts | 19 +++++++-- .../chat/openai-responses/items/store.ts | 2 +- .../chat/openai-responses/pipeline.ts | 24 ++++------- .../server-tools/image-generation.ts | 9 ++-- .../chat/{interceptors.ts => rules.ts} | 41 +++++++++---------- .../gateway/src/data-plane/chat/stages.ts | 2 +- 17 files changed, 80 insertions(+), 81 deletions(-) rename packages/gateway/__tests__/data-plane/{chat-interceptors_test.ts => chat-rules_test.ts} (99%) rename packages/gateway/src/data-plane/chat/{interceptors.ts => rules.ts} (97%) diff --git a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts b/packages/gateway/__tests__/data-plane/chat-rules_test.ts similarity index 99% rename from packages/gateway/__tests__/data-plane/chat-interceptors_test.ts rename to packages/gateway/__tests__/data-plane/chat-rules_test.ts index 279aed3eca..a4dba3a1eb 100644 --- a/packages/gateway/__tests__/data-plane/chat-interceptors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-rules_test.ts @@ -30,7 +30,7 @@ import { vendorKimiNormalizeForOpenAIChatCompletions, vendorQwenNormalizeForOpenAIChatCompletions, vendorQwenNormalizeForOpenAIResponses, -} from '../../src/data-plane/chat/interceptors.ts'; +} from '../../src/data-plane/chat/rules.ts'; import type { AttemptSelector } from '../../src/data-plane/pipeline/facts.ts'; import { compose, defineStage, move, run } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts index 1b3fd9273a..48509abe65 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts @@ -132,10 +132,10 @@ beforeEach(() => { }); describe('the chat completions chain', () => { - // The interceptors rewrite the request as a fact, so what they rewrite has to be what the + // The wire's rules rewrite the request as a fact, so what they rewrite has to be what the // ending sends. An ending that asked the resolver for the payload instead would read one no - // interceptor had touched, and every rewrite in the chain would go nowhere. - it('sends what the interceptors rewrote, not what the resolver materialized', async () => { + // rule had touched, and every rewrite in the chain would go nowhere. + it('sends what the rules rewrote, not what the resolver materialized', async () => { let sent: { messages: { role: string }[] } | undefined; affinityPayload = { ...payload, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts index 872fbba320..fa3823c657 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts @@ -105,6 +105,6 @@ export const driveServerToolStage = (registrations: readonly ServerToolRegistrat } as never); // The reading the run hands up, which is where every turn's identity and cost land now - // that a stage rather than an interceptor observes them. + // that a stage observes them. return { ...resultOf(facts), reading: facts[STREAMED_USAGE] as Deferred | null }; }; diff --git a/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts index 6517ea0a8c..1d3ce77585 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts @@ -2,7 +2,7 @@ // // Everything else in this directory calls a chain, a stage or a helper directly. What no such // test can state is that the route *works*: the parts it skips — the provider's own boundary -// interceptors, the fetcher, the epilogue that writes the client's bytes and settles what the +// boundary rules, the fetcher, the epilogue that writes the client's bytes and settles what the // stream billed — are exactly where a turn has been lost before. So each row here drives the // real Hono route against a Copilot upstream, reads the bytes the client would read, and says // something about them that a broken chain could not produce. diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index e038579132..92c6d413a0 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -9,7 +9,7 @@ // Beside the matrix, three things that are about the fork rather than about a pair: that // failover moves from a native candidate onto a translated one, that a pair's own refusal // rewrite survives the trip, and that a rule which speaks about one protocol's wire does not -// run on a turn that leaves for another — which is what the interceptor form said with +// run on a turn that leaves for another — which is what the replaced surface said with // `ctx.targetApi !== ` and what position says here. import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -512,7 +512,7 @@ describe('a rule that speaks about one protocol-s wire', () => { // And a rule the *other* wire owns stays off this turn. `stream_options` is an OpenAI Chat // Completions field, so an OpenAI Chat Completions turn dialled over Anthropic Messages must // not carry it — - // which is what the interceptor form said with `ctx.targetApi !== 'openaiChatCompletions'` and + // which is what the replaced surface said with a target-API guard and // what position says here. it('does not carry another wire-s request rules onto the wire it dialled', async () => { let sent: Record | undefined; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts index f42b4a7a8b..8c3d236ad2 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts @@ -28,7 +28,7 @@ import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols // "code":"invalid_request_body"}}`. // // The gateway answers the probe itself instead. That is honest for what the -// probe actually asks: by the time this interceptor runs, model resolution has +// probe actually asks: by the time the stage that answers it runs, model resolution has // already picked a real candidate for the requested id, so an id no upstream // serves still fails at the serve layer with a 404 and the CLI still reports // it not found. What we suppress is only the pointless one-token generation diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index 422c36015a..87e0b6baf1 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -42,7 +42,7 @@ import { applyRoleCompatibilityToAnthropicMessages, disableReasoningOnForcedToolChoiceForAnthropicMessages, stripBillingAttributionFromAnthropicMessages, -} from '../interceptors.ts'; +} from '../rules.ts'; import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { materializeAttempt, resolveChatCandidates, type ChatNarrowing, type ChatServices } from '../stages.ts'; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 88d00ff0d9..05e018f6d8 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -14,11 +14,10 @@ // // `/v1/messages/count_tokens` is not one of them: it is a second operation over this // protocol rather than another wire under this pipeline, so it is a chain of its own in -// `count-tokens.ts`. The web-search shim is not one of them either, and for a worse reason: it -// never became a stage and the interceptor array that ran it is gone, so a turn declaring -// Anthropic's native `web_search` reaches the upstream unshimmed and the flag that gates it does -// nothing. Only its request half survives on a live path, in the counting chain. The code is kept -// so that porting it starts from something rather than from nothing. +// `count-tokens.ts`. Anthropic's native `web_search` is not a wire either: an upstream that +// cannot serve it gets the declaration rewritten and each search executed here, which is +// `runAnthropicMessagesWebSearchTool` below — one stage, because Anthropic carries the tool +// result inside the same turn and there is nothing to ask again for. import { wrapAnthropicMessagesAffinityEgress } from './affinity/egress.ts'; import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; @@ -39,14 +38,14 @@ import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts' import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; +import { meterChatWire } from '../meter.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; +import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; import { applyRoleCompatibilityToAnthropicMessages, disableReasoningOnForcedToolChoiceForAnthropicMessages, stripBillingAttributionFromAnthropicMessages, -} from '../interceptors.ts'; -import { meterChatWire } from '../meter.ts'; -import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; -import { openaiResponsesWire } from '../openai-responses/pipeline.ts'; +} from '../rules.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; @@ -195,7 +194,7 @@ const renderSSE = (frames: AsyncIterable openaiResponsesTarget.pick(candidate.model.endpoints), + }), dialOpenAIResponsesCompaction, ]); }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts index d4da3e389b..116e8448e5 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts @@ -37,7 +37,7 @@ export interface StatefulOpenAIResponsesStore { persistOutputItem(row: StoredOpenAIResponsesItem): Promise; commitSnapshot(responseId: string, mode: OpenAIResponsesSnapshotMode, outputItemIds: readonly string[]): Promise; // Per-attempt transient state. `beginAttempt` reseeds the private-payload - // scratchpad from hydrated items; interceptors can add server-only state + // scratchpad from hydrated items; a server tool can add server-only state // during the turn and output capture persists it with the exact wire item. beginAttempt(privatePayloads: ReadonlyMap): void; registerPrivatePayload(id: string, privatePayload: unknown): void; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 46703b6ad1..8c205f536f 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -9,14 +9,15 @@ // beginStoredAttempt reseeds the store's per-attempt scratchpad // expandShimCompactions a compaction this gateway wrote, back into what it stood for // summarizeForCompaction answers a turn that asked for a compaction with one +// runOpenAIResponsesServerTools a hosted tool the upstream cannot serve, emulated here // dialChatWire the ending: picks this candidate's wire and hands into it // // Three wires, all handing up `response.chat.openaiResponses`: this protocol's own, and the two // translated ones — OpenAI Responses via Anthropic Messages and via OpenAI Chat Completions — each a // handoff followed by that protocol's own wire. What sits *in* a wire rather than above the // fork is a rule that speaks about that wire: the role rewrite and the cache-bucket fold both -// do, which is why their interceptor forms stood down on `ctx.targetApi !== 'openaiResponses'`, and -// position says it here instead. +// do, and position is what says so — a stage below the fork runs only on the wire the fork +// chose, so neither needs a guard about which protocol it is looking at. // // Two transports enter here — `POST /v1/responses` and the WebSocket one — and the only thing // they disagree about is how a streamed answer is framed, which is what `framing` says. Below @@ -32,17 +33,10 @@ // compaction routes and is rewritten exactly as a turn is. The dial here asks for // `generate`; what the ending answers with is the branch the provider says it ran, which // is why the envelope a compaction is arrives somewhere rather than nowhere. -// - the server-tool shims. They never became stages and the interceptor array that ran them -// is gone, so `withOpenAIResponsesServerToolShim` and the two tools under it are reachable only -// from their own tests: a turn declaring a hosted `web_search` or `image_generation` tool -// reaches the upstream unshimmed, and the two flags that gate them do nothing. The code is -// kept so that porting it starts from something rather than from nothing. Nothing in a -// pipelined turn writes to the store's private-payload scratchpad for the same reason — -// the shim that writes to it does not run. // -// One deliberate difference from `respond.ts`, shared with every other family on the -// pipeline: an upstream that refused is answered in its own words, with the status it sent, -// rather than being quoted back inside an envelope this gateway wrote. +// One deliberate difference from the surface this replaced, shared with every other family: an +// upstream that refused is answered in its own words, with the status it sent, rather than +// being quoted back inside an envelope this gateway wrote. import { analyzeOpenAIResponsesAffinity } from './affinity/ingress.ts'; import { wrapOpenAIResponsesClientEgress } from './client-output.ts'; @@ -77,6 +71,8 @@ import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import { anthropicMessagesWire } from '../anthropic-messages/pipeline.ts'; import type { ChatFacts } from '../facts.ts'; import { dialChatWire, handOff, type ChatWire } from '../handoff.ts'; +import { meterChatWire } from '../meter.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; import { applyRoleCompatibilityToOpenAIResponses, disableReasoningOnForcedToolChoiceForOpenAIResponses, @@ -84,9 +80,7 @@ import { stripPromptCacheKeyForOpenAIResponses, vendorDeepSeekNormalizeForOpenAIResponses, vendorQwenNormalizeForOpenAIResponses, -} from '../interceptors.ts'; -import { meterChatWire } from '../meter.ts'; -import { openaiChatCompletionsWire } from '../openai-chat-completions/pipeline.ts'; +} from '../rules.ts'; import { applyRulesToUpstreamOpenAIResponses } from '../shared/alias-rules.ts'; import { tryCatchChatServeFailure } from '../shared/errors.ts'; import { createExternalImageLoader } from '../shared/external-image-loader.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts index 4a6c6bb81a..c88b4a2e7e 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts @@ -888,11 +888,10 @@ const errorFromBody = (body: string, status: number): { type?: string; code: str }; // Per-request inputs the dispatcher's backend call needs. Captured in the -// registration closure from `ctx`/`request` so the dispatcher stays free of the -// interceptor signature. Edit sources are NOT captured here — they are -// re-collected from the live `ctx.payload.input` at dispatch time so an image -// generated in an earlier turn (fed back as an `input_image`) becomes editable -// in a later turn. `imageDispatchCount` bounds how many real backend image +// registration closure so the dispatcher reads none of the turn it was +// registered for. Edit sources are NOT captured here — they are re-collected +// from the payload the current descent is sending, so an image generated in an +// earlier turn (fed back as an `input_image`) becomes editable in a later one. `imageDispatchCount` bounds how many real backend image // calls one response may issue. interface ShimState { config: MaterializedImageGenerationConfig; diff --git a/packages/gateway/src/data-plane/chat/interceptors.ts b/packages/gateway/src/data-plane/chat/rules.ts similarity index 97% rename from packages/gateway/src/data-plane/chat/interceptors.ts rename to packages/gateway/src/data-plane/chat/rules.ts index e4044c6edb..fbd60d7eb8 100644 --- a/packages/gateway/src/data-plane/chat/interceptors.ts +++ b/packages/gateway/src/data-plane/chat/rules.ts @@ -1,23 +1,22 @@ -// The chat interceptors, as stages. Each is written against one protocol's payload key and -// dropped into that protocol's array — one stage per (interceptor, protocol) pair, because -// the payload types differ, but one *rule* each, written once here: the role fold, the flag -// reading and the key removal are shared, and only the walk over a protocol's own items is -// per protocol. +// The rules every chat chain composes. Each is written against one protocol's payload key and +// dropped into that protocol's array — one stage per (rule, protocol) pair, because the payload +// types differ, but one *rule* each, written once here: the role fold, the flag reading and the +// key removal are shared, and only the walk over a protocol's own items is per protocol. // -// What changes from the interceptor form is the three things the pipeline replaced. A stage -// returns the record it hands on instead of assigning to `ctx.payload`, so a rewrite is a -// spread and structural sharing is what makes it cheap — every rule here writes -// conditionally, which is the convention that keeps a 49-message conversation costing three -// objects, and it is also what lets a Gemini generateContent payload be rewritten at all now that the record -// is frozen and `delete ctx.payload.x` would throw. `ctx.targetApi` is a declared need rather -// than an ambient field — array position says which chain a stage is in, and what it reads -// about the protocol it is looking at is in the record. The guards themselves are not deleted -// by that, they are split: a guard about the *wire* is said by position, because a stage below -// the fork runs only on the wire the fork chose, and a guard about what the *client* spoke is -// said by needing `ingress.chat.sourceProtocol`, which survives the switch. And a rule that reads the response — the two that rewrite a stream — states -// it as a response-direction declaration rather than as work after `await run()`, with what it -// learned on the way down carried in the stage's own closure, because a response-side `needs` -// can only name what the ending provides. +// A rule states three things by declaration that the surface this replaced stated by procedure. +// It returns the record it hands on rather than assigning to a mutable payload, so a rewrite is +// a spread and structural sharing is what makes it cheap — every rule here writes conditionally, +// which is the convention that keeps a 49-message conversation costing three objects, and is +// also what lets a Gemini generateContent payload be rewritten at all now that the record is +// frozen and deleting a key from it would throw. Which protocol a rule is looking at is a +// declared need rather than an ambient field: array position says which chain a stage is in, and +// what it reads about the turn is in the record. The guards are not deleted by that, they are +// split — a guard about the *wire* is said by position, because a stage below the fork runs only +// on the wire the fork chose, and a guard about what the *client* spoke is said by needing +// `ingress.chat.sourceProtocol`. And a rule that reads the response — the two that rewrite a +// stream — states it as a response-direction declaration rather than as work after the descent, +// with what it learned on the way down carried in the stage's own closure, because a +// response-side `needs` can only name what the ending provides. import type { Chat, ChatAnswer } from './facts.ts'; import { asJsonObject, type JsonObject, readJsonNumber } from '../../shared/json-helpers.ts'; @@ -945,8 +944,8 @@ const scrubbedSystemPrompt = (system: AnthropicMessagesPayload['system']): Anthr // The three strippers below are unconditional rather than flag-gated, and that is a statement // about the target graph rather than about any upstream: Gemini generateContent has no wire of its own here, // so every turn is served through a translation, and what no translation can carry cannot be -// sent whichever candidate answers. What was `delete ctx.payload.x` is a rewrite now — the -// record is frozen, so a stage that deleted in place would throw rather than strip. +// sent whichever candidate answers. Stripping is a rewrite rather than a deletion — the record +// is frozen, so a stage that deleted a key in place would throw rather than strip. /** Gemini generateContent file and code parts have no equivalent anywhere the translations reach, so they go * at source and every target sees translatable parts. A part left holding nothing goes with diff --git a/packages/gateway/src/data-plane/chat/stages.ts b/packages/gateway/src/data-plane/chat/stages.ts index a32c048f55..b8748699f6 100644 --- a/packages/gateway/src/data-plane/chat/stages.ts +++ b/packages/gateway/src/data-plane/chat/stages.ts @@ -123,7 +123,7 @@ export const resolveChatCandidates = (narrowing: ChatNar * Affinity materializes one per candidate — client-carried state rewritten for the upstream * that will see it — and it has to enter the record here rather than be read by the stage * that dials, because everything between this stage and the dial rewrites the request as a - * fact. An ending that asked the resolver instead would be reading a payload no interceptor + * fact. An ending that asked the resolver instead would be reading a payload no rule * had touched, and the rewrites would go nowhere. * * It sits below the fork because there is no single payload above it: each candidate has its From 77ff87f7c9c32b1ee7f7fcb8fbc1b0cd4c0ab8c4 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 11:13:42 +0800 Subject: [PATCH 092/156] refactor(dump): delete the record shape nothing produces any more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every route opens its run through the prologue, so no turn has been recorded as its two edges since the last family moved. What was left was the shape itself: declared in the types, written by an accumulator nothing constructed, branched on by the store, the wire codec and the dashboard, and carried by stored rows. It is deleted rather than kept readable. An edge record holds what the client sent and what it got back and nothing about what happened between, so there is no run to rebuild it into; and a dump record is diagnostic and already expires under its key's own retention, so migration `0084` brings a deletion forward rather than inventing one. The delete trigger retires the files with the rows. With one shape left, the `shape` discriminator goes too — a record is its metadata and its NDJSON — and so does everything that existed to serve the other one: the prepared-request-body seam, the header and stream-event codecs, the request and response halves, and the dashboard's header table with the redactor behind it. The collected view had to be rebuilt rather than dropped. It was fed by the edge record's own frame log; a run holds its frames as `stream.frame` events written against the run's object space, so reading them needs a reader for that space — which the format described and nothing implemented. `createRunReader` is it: it folds nodes as they arrive and resolves `{"$": n}` against them, which it can do line by line because a reference never points forward. The dashboard collects the first stream a run opened, which is the one its edge teed for the client. `FileDumpStore.get` now raises on a row whose descriptor names anything but a run. After the migration there are none, so such a row is corruption — and a reader that returned half a record would report it as a turn that served nothing. --- .../requests/body-rendering_test.ts | 149 ---------- .../components/requests/detail_test.tsx | 40 +-- .../requests/header-redaction_test.ts | 45 --- apps/web/package.json | 1 + .../src/components/requests/body-render.ts | 78 +---- apps/web/src/components/requests/detail.tsx | 192 +++---------- .../src/components/requests/header-redact.ts | 15 - .../web/src/components/requests/run-stream.ts | 40 +++ apps/web/src/i18n/locales/en.ts | 12 - apps/web/src/i18n/locales/zh-Hans.ts | 12 - .../src/routes/dashboard-monitor-requests.tsx | 7 +- .../__tests__/control-plane/dump_test.ts | 22 +- .../openai-completions/http_test.ts | 7 +- .../data-plane/shared/gateway-ctx_test.ts | 22 +- .../gateway/__tests__/dump/run-sink_test.ts | 9 +- .../gateway/__tests__/dump/test-fixtures.ts | 39 +-- packages/gateway/__tests__/dump/wire_test.ts | 129 +-------- .../gateway/__tests__/repo/dump-store_test.ts | 263 ++++------------- .../scheduled/expiration-sweeps_test.ts | 15 +- packages/gateway/__tests__/scheduled_test.ts | 16 +- packages/gateway/__tests__/vitest.setup.ts | 1 - .../0084_drop_edge_dump_records.sql | 20 ++ .../src/data-plane/shared/gateway-ctx.ts | 21 +- packages/gateway/src/dump/accumulator.ts | 271 ------------------ packages/gateway/src/dump/run-sink.ts | 1 - packages/gateway/src/dump/store-contract.ts | 6 +- packages/gateway/src/dump/types.ts | 103 +------ packages/gateway/src/dump/wire.ts | 70 +---- packages/gateway/src/repo/dump-store.ts | 134 +-------- packages/pipeline/src/dump.ts | 57 ++++ packages/pipeline/src/index.ts | 1 + pnpm-lock.yaml | 3 + 32 files changed, 322 insertions(+), 1479 deletions(-) delete mode 100644 apps/web/__tests__/components/requests/body-rendering_test.ts delete mode 100644 apps/web/__tests__/components/requests/header-redaction_test.ts delete mode 100644 apps/web/src/components/requests/header-redact.ts create mode 100644 apps/web/src/components/requests/run-stream.ts create mode 100644 packages/gateway/migrations/0084_drop_edge_dump_records.sql delete mode 100644 packages/gateway/src/dump/accumulator.ts diff --git a/apps/web/__tests__/components/requests/body-rendering_test.ts b/apps/web/__tests__/components/requests/body-rendering_test.ts deleted file mode 100644 index 1fa698998d..0000000000 --- a/apps/web/__tests__/components/requests/body-rendering_test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { contentTypeOf, EMPTY_BODY, renderBody } from '../../../src/components/requests/body-render'; - -// The dump carries bodies as they went over the wire, so a base64 fixture is -// built from bytes rather than from a string literal. -const base64 = (bytes: string) => btoa(bytes); - -const withoutTypedArrayBase64 = (run: () => T): T => { - const fromBase64 = Object.getOwnPropertyDescriptor(Uint8Array, 'fromBase64'); - const toBase64 = Object.getOwnPropertyDescriptor(Uint8Array.prototype, 'toBase64'); - Object.defineProperty(Uint8Array, 'fromBase64', { configurable: true, value: undefined }); - Object.defineProperty(Uint8Array.prototype, 'toBase64', { configurable: true, value: undefined }); - try { - return run(); - } finally { - if (fromBase64) Object.defineProperty(Uint8Array, 'fromBase64', fromBase64); - else Reflect.deleteProperty(Uint8Array, 'fromBase64'); - if (toBase64) Object.defineProperty(Uint8Array.prototype, 'toBase64', toBase64); - else Reflect.deleteProperty(Uint8Array.prototype, 'toBase64'); - } -}; - -describe('content type lookup', () => { - it('finds the header however it was capitalized, and says nothing when it is absent', () => { - expect(contentTypeOf([['Content-Type', 'application/json']])).toBe('application/json'); - expect(contentTypeOf([['content-length', '12']])).toBe(''); - }); -}); - -describe('body rendering', () => { - it('shows nothing for a body that carries no data', () => { - expect(renderBody({ data: '', encoding: 'utf8' }, 'application/json')).toEqual(EMPTY_BODY); - }); - - it('pretty-prints JSON and says that is what it is', () => { - const rendered = renderBody({ data: '{"model":"gpt-5","stream":true}', encoding: 'utf8' }, 'application/json'); - - expect(rendered.isJson).toBe(true); - expect(rendered.text).toBe('{\n "model": "gpt-5",\n "stream": true\n}'); - expect(rendered.copyText).toBe(rendered.text); - }); - - it('leaves a body that is not JSON exactly as it arrived', () => { - const rendered = renderBody({ data: 'data: [DONE]\n\n', encoding: 'utf8' }, 'text/event-stream'); - - expect(rendered.isJson).toBe(false); - expect(rendered.text).toBe('data: [DONE]\n\n'); - expect(rendered.decodeError).toBeNull(); - }); - - it('decodes a base64 body before reading it as JSON', () => { - const rendered = renderBody({ data: base64('{"ok":true}'), encoding: 'base64' }, 'application/json'); - - expect(rendered.isJson).toBe(true); - expect(rendered.text).toBe('{\n "ok": true\n}'); - }); - - // A body that is not text at all keeps its base64 on screen rather than - // becoming replacement characters, and says why. - it('reports a body it cannot decode instead of showing mojibake', () => { - const data = base64('ÿþý'); - - const rendered = renderBody({ data, encoding: 'base64' }, 'application/octet-stream'); - - expect(rendered.decodeError).toBeTruthy(); - expect(rendered.text).toBe(data); - expect(rendered.copyText).toBe(data); - expect(rendered.isJson).toBe(false); - }); -}); - -describe('multipart body rendering', () => { - const boundary = 'floway-boundary'; - const bytes = (...values: number[]) => String.fromCharCode(...values); - const part = (headers: string, body: string) => `${headers}\r\n\r\n${body}`; - const multipart = (parts: string[]) => - [`--${boundary}`, parts.join(`\r\n--${boundary}\r\n`), `--${boundary}--`, ''].join('\r\n'); - - const jsonPart = part('content-disposition: form-data; name="model"\r\ncontent-type: Application/Vnd.Floway+JSON; charset=utf-8', '{"id":1}'); - const textWire = multipart([jsonPart, part('content-disposition: form-data; name="note"', 'hello')]); - const render = (wire: string, contentType = `multipart/form-data; boundary=${boundary}`) => - renderBody({ data: base64(wire), encoding: 'base64' }, contentType); - - it('shows every textual part as text rather than as one base64 blob', () => { - const rendered = render(textWire); - - expect(rendered.text).toContain('{"id":1}'); - expect(rendered.text).toContain('hello'); - expect(rendered.text).toContain(`--${boundary}--`); - expect(rendered.isJson).toBe(false); - }); - - // Bytes no textual decoding could survive, so a render that reached the - // placeholder cannot have come from the plain-body fallback. - const image = bytes(0x00, 0x01, 0xFF); - const binaryWire = multipart([jsonPart, part('content-disposition: form-data; name="file"\r\ncontent-type: image/png', image)]); - - it('stands in for a part that is not text', () => { - const rendered = render(binaryWire); - - expect(rendered.text).toContain('{"id":1}'); - expect(rendered.text).toContain('[binary, 3 bytes, content-type=image/png]'); - expect(rendered.text).toContain(base64(image)); - expect(rendered.text).not.toContain(image); - }); - - it('renders binary parts directly through the legacy-browser fallback', () => { - const rendered = withoutTypedArrayBase64(() => render(binaryWire)); - - expect(rendered.text).toContain('[binary, 3 bytes, content-type=image/png]'); - expect(rendered.text).toContain(base64(image)); - }); - - it('copies the body as it arrived rather than as it is shown', () => { - const data = base64(textWire); - - expect(renderBody({ data, encoding: 'base64' }, `multipart/form-data; boundary=${boundary}`).copyText).toBe(data); - expect(renderBody({ data, encoding: 'base64' }, `multipart/form-data; boundary=${boundary}`).text).not.toBe(data); - }); - - it('reads the boundary out of a quoted parameter too', () => { - expect(render(binaryWire, `multipart/form-data; boundary="${boundary}"`).text).toContain('[binary, 3 bytes'); - }); - - it('falls back to the plain body when the content type names no boundary', () => { - const rendered = render(textWire, 'multipart/form-data'); - - expect(rendered.text).toBe(textWire); - expect(rendered.text).not.toContain('[binary,'); - }); - - // A PNG signature leads with 0x89, inside the 0x80-0x9F range where the - // windows-1252 decoder the Encoding Standard resolves latin1 to - // (https://encoding.spec.whatwg.org/#names-and-labels) is not byte-transparent. - // Reading the wire through any text encoding loses those bytes and abandons - // the whole multipart render, so every PNG upload arrives as a raw blob. - const png = bytes(0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A); - const pngWire = multipart([jsonPart, part('content-disposition: form-data; name="file"\r\ncontent-type: image/png', png)]); - - it('round-trips a PNG signature rather than losing it to the encoding', () => { - const rendered = render(pngWire); - - expect(rendered.text).toContain('{"id":1}'); - expect(rendered.text).toContain('[binary, 8 bytes, content-type=image/png]'); - expect(rendered.text).toContain(base64(png)); - expect(rendered.decodeError).toBeNull(); - }); -}); diff --git a/apps/web/__tests__/components/requests/detail_test.tsx b/apps/web/__tests__/components/requests/detail_test.tsx index dd158c1457..15712c7567 100644 --- a/apps/web/__tests__/components/requests/detail_test.tsx +++ b/apps/web/__tests__/components/requests/detail_test.tsx @@ -21,24 +21,7 @@ const meta: DumpMetadata = { error: null, }; -const edgeRecord: DumpRecord = { - shape: 'edge', - meta, - request: { - method: 'POST', - path: '/v1/embeddings', - headers: [['content-type', 'application/json']], - body: { encoding: 'utf8', data: '{"input":"hi"}' }, - }, - response: { - status: 200, - headers: [['content-type', 'application/json']], - body: { type: 'bytes', body: { encoding: 'utf8', data: '{"object":"list"}' } }, - }, -}; - const runRecord: DumpRecord = { - shape: 'run', meta, events: '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n' + '{"type":"object","fromObjectId":1,"nodes":[{"model":"text-embedding-3-small"}]}\n' @@ -48,33 +31,24 @@ const runRecord: DumpRecord = { const panel = (record: DumpRecord) => renderInApp(); -// The shape follows the endpoint, so the panel is handed both and has to tell -// them apart: an endpoint on the onion is recorded as its two edges, a pipelined -// one as the whole run. +// A record is a whole run — every stage, both directions — and the panel draws its event +// stream, which is the whole of what it holds. describe('request detail panel', () => { - it('draws the two edges of an edge-shaped record', () => { - const { container } = panel(edgeRecord); - const headings = [...container.querySelectorAll('h3')].map(node => node.textContent); - expect(headings).toEqual(['Request', 'Request body', 'Response', 'Response body']); - expect(container.textContent).toContain('"input": "hi"'); - expect(container.textContent).toContain('"object": "list"'); - }); - - it('draws the event stream of a run-shaped record', () => { + it('draws the event stream a record holds', () => { const { container } = panel(runRecord); const headings = [...container.querySelectorAll('h3')].map(node => node.textContent); - expect(headings).toEqual(['Run']); + // The run's own events, and the one value the frames in them add up to. + expect(headings).toEqual(['Run', 'Collected']); // One block per NDJSON line, each labelled with the event's own kind. expect([...container.querySelectorAll('pre')]).toHaveLength(3); expect(container.textContent).toContain('stage.entered'); expect(container.textContent).toContain('response.http.status'); - // Nothing from the edge shape leaks into it: a run has no header tables and - // no separate request body. + // The stream is the whole panel: there are no header tables and no separate body section. expect(container.querySelector('table')).toBe(null); }); it('says so when a run recorded no events at all', () => { - const { container } = panel({ shape: 'run', meta, events: '' }); + const { container } = panel({ meta, events: '' }); expect(container.textContent).toContain('This run recorded no events.'); }); }); diff --git a/apps/web/__tests__/components/requests/header-redaction_test.ts b/apps/web/__tests__/components/requests/header-redaction_test.ts deleted file mode 100644 index 9ca277b604..0000000000 --- a/apps/web/__tests__/components/requests/header-redaction_test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { isSensitiveHeader, redactHeaderValue } from '../../../src/components/requests/header-redact'; - -describe('header redaction', () => { - it('knows a credential header however it was capitalized on the wire', () => { - expect(isSensitiveHeader('Authorization')).toBe(true); - expect(isSensitiveHeader('COOKIE')).toBe(true); - expect(isSensitiveHeader('Proxy-Authorization')).toBe(true); - expect(isSensitiveHeader('Set-Cookie')).toBe(true); - expect(isSensitiveHeader('X-Api-Key')).toBe(true); - expect(isSensitiveHeader('X-Goog-Api-Key')).toBe(true); - }); - - it('leaves a header that carries no credential alone', () => { - expect(isSensitiveHeader('content-type')).toBe(false); - expect(isSensitiveHeader('x-floway-session')).toBe(false); - }); - - it('shows nothing at all of a short value', () => { - expect(redactHeaderValue('')).toBe(''); - expect(redactHeaderValue('Bearer sk-1')).toBe('•'.repeat(11)); - expect(redactHeaderValue('0123456789abcdef')).toBe('•'.repeat(16)); - }); - - // One character past the fully-masked window the mask keeps both ends, which - // leaves sixteen of the seventeen characters legible. Recorded in - // `data/backlog.md`: no credential this app shows is that short, but the - // window is what decides it. - it('starts keeping the ends one character past the fully-masked window', () => { - expect(redactHeaderValue('0123456789abcdefg')).toBe('01234567•9abcdefg'); - }); - - it('keeps the ends of a long value so it can be told apart from another', () => { - const value = `sk-ant-${'x'.repeat(40)}-tail`; - - const redacted = redactHeaderValue(value); - - expect(redacted).toHaveLength(value.length); - expect(redacted.startsWith('sk-ant-x')).toBe(true); - expect(redacted.endsWith('xxxx-tail')).toBe(false); - expect(redacted.endsWith('xxx-tail')).toBe(true); - expect(redacted.slice(8, -8)).toBe('•'.repeat(value.length - 16)); - }); -}); diff --git a/apps/web/package.json b/apps/web/package.json index e64c61288b..9943349462 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -11,6 +11,7 @@ "typecheck": "react-router typegen && tsc --noEmit && tsc --noEmit -p tsconfig.scripts.json" }, "dependencies": { + "@floway-dev/pipeline": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", "@floway-dev/provider-custom": "workspace:*", diff --git a/apps/web/src/components/requests/body-render.ts b/apps/web/src/components/requests/body-render.ts index d81c7f4a78..3a1e0b451b 100644 --- a/apps/web/src/components/requests/body-render.ts +++ b/apps/web/src/components/requests/body-render.ts @@ -1,7 +1,5 @@ -import { decodeWebBase64, decodeWebBase64BinaryString, encodeBase64BinaryString } from '../../lib/base-encoding'; -import { errorMessage } from '../../lib/error-message'; -import type { DumpBody } from '@floway-dev/gateway/dump-types'; -import { isTextualMediaType, parseMediaType } from '@floway-dev/protocols/common'; +// What a code block renders: the text, what a copy button copies, and the failure to name if +// the text could not be produced. export interface RenderedBody { text: string; @@ -9,75 +7,3 @@ export interface RenderedBody { decodeError: string | null; isJson: boolean; } - -export const EMPTY_BODY: RenderedBody = { text: '', copyText: '', decodeError: null, isJson: false }; - -export const contentTypeOf = (headers: Array<[string, string]>): string => { - return headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''; -}; - -export const renderBody = (body: DumpBody, contentType: string): RenderedBody => { - if (!body.data) return EMPTY_BODY; - if (parseMediaType(contentType)?.type === 'multipart' && body.encoding === 'base64') { - const multipart = renderMultipart(body.data, contentType); - if (multipart !== null) { - return { text: multipart, copyText: body.data, decodeError: null, isJson: false }; - } - } - - let text = body.data; - if (body.encoding === 'base64') { - try { - const bytes = decodeWebBase64(body.data); - text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); - } catch (error) { - return { - text: body.data, - copyText: body.data, - decodeError: errorMessage(error), - isJson: false, - }; - } - } - - try { - const pretty = JSON.stringify(JSON.parse(text) as unknown, null, 2); - return { text: pretty, copyText: pretty, decodeError: null, isJson: true }; - } catch { - return { text, copyText: text, decodeError: null, isJson: false }; - } -}; - -const renderMultipart = (base64: string, contentType: string): string | null => { - const boundaryMatch = /;\s*boundary=(?:"([^"]+)"|([^;\s]+))/i.exec(contentType); - const boundary = boundaryMatch?.[1] ?? boundaryMatch?.[2]; - if (!boundary) return null; - - try { - // The wire stays a binary string with one code unit per byte, so slicing it - // and encoding a part back through the byte codec round-trips. Decoding it - // through a text encoding first does not: the Encoding Standard resolves - // latin1 to windows-1252 (https://encoding.spec.whatwg.org/#names-and-labels), - // which maps 0x80-0x9F -- a PNG signature leads with 0x89 -- to code points - // above 255. - const wire = decodeWebBase64BinaryString(base64); - const chunks = wire.split(`--${boundary}`); - if (chunks.length < 3) return null; - - const rendered = chunks.slice(1, -1).map(chunk => { - const part = chunk.replace(/^\r\n/, '').replace(/\r\n$/, ''); - const separator = part.indexOf('\r\n\r\n'); - if (separator < 0) throw new Error('Malformed multipart body'); - const headers = part.slice(0, separator); - const data = part.slice(separator + 4); - const type = /^content-type:\s*(.+)$/im.exec(headers)?.[1]?.trim() ?? ''; - const textual = !type || isTextualMediaType(type); - if (textual) return `${headers}\r\n\r\n${new TextDecoder().decode(Uint8Array.from(data, c => c.charCodeAt(0)))}`; - const encoded = encodeBase64BinaryString(data).replace(/.{76}(?=.)/g, '$&\n'); - return `${headers}\r\n\r\n[binary, ${data.length} bytes, content-type=${type}]\r\n${encoded}`; - }); - return `--${boundary}\r\n${rendered.join(`\r\n--${boundary}\r\n`)}\r\n--${boundary}--\r\n`; - } catch { - return null; - } -}; diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index 4ce5725aea..fd6ac08e4c 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -1,20 +1,10 @@ -import { - EyeOffRegular, - EyeRegular, -} from '@fluentui/react-icons'; import { useMemo, useState } from 'react'; import type { PropsWithChildren, ReactNode } from 'react'; -import { contentTypeOf, EMPTY_BODY, renderBody, type RenderedBody } from './body-render'; +import { type RenderedBody } from './body-render'; import { errorLabel, requestSeverity } from './format'; -import { isSensitiveHeader, redactHeaderValue } from './header-redact'; import { renderRunEvents } from './run-render'; -import { - detectCollectKind, - renderStreamEvents, - streamEventsCopyText, - type CollectedStream, -} from './stream-render'; +import { detectCollectKind, type CollectedStream } from './stream-render'; import { fluentComponents } from '../../fluent'; import { useTranslation } from '../../i18n/translation'; import { useDangerTextClass } from '../ui/danger'; @@ -25,9 +15,9 @@ import { highlight, prismTokenStyles } from '../ui/prism'; import { ScrollArea } from '../ui/scroll-area'; import { TooltipIconButton } from '../ui/tooltip-icon-button'; import { copyOutcomeIcon, useCopyLabel, useCopyToClipboard } from '../ui/use-copy-to-clipboard'; -import type { DumpRecord, DumpStreamEvent } from '@floway-dev/gateway/dump-types'; +import type { DumpRecord } from '@floway-dev/gateway/dump-types'; -const { Tab, TabList, Text, makeStyles, mergeClasses } = fluentComponents; +const { Text, makeStyles, mergeClasses } = fluentComponents; // Region dividers take WinUI's divider brush (`colorNeutralStroke3`), not the // control outline; the two only differ in the dark dictionary. @@ -116,40 +106,6 @@ function CodeView({ body }: { body: RenderedBody }) { return
; } -function HeaderTable({ headers }: { headers: Array<[string, string]> }) { - const { t } = useTranslation(); - const s = useStyles(); - const [revealed, setRevealed] = useState>(new Set()); - return ( - - {headers.map(([name, value], index) => { - const sensitive = isSensitiveHeader(name); - const visible = revealed.has(index); - return ( - - - - - ); - })} -
{name} - {sensitive && !visible ? redactHeaderValue(value) : value} - {sensitive && ( - : } - label={visible ? t('dashboard.requests.hideValue') : t('dashboard.requests.revealValue')} - onClick={() => setRevealed(current => { - const next = new Set(current); - if (next.has(index)) next.delete(index); else next.add(index); - return next; - })} - /> - )} -
- ); -} - // Deliberately not `SectionHeader`: this bar is sticky and holds a fixed // height, which that primitive's narrow-viewport rule would stack away, and it // carries a detail slot beside the title that the primitive has no prop for. @@ -162,10 +118,6 @@ function SectionBody({ children }: PropsWithChildren) { return {children}; } -function HeaderSectionBody({ children }: PropsWithChildren) { - return {children}; -} - export function RequestDetailPanel({ collected: loadedCollected, error: loadedError, record: loadedRecord, recordId: selectedRecordId, retainLastRecord }: { collected: CollectedStream | null; error: string | null; @@ -177,7 +129,6 @@ export function RequestDetailPanel({ collected: loadedCollected, error: loadedEr const { t } = useTranslation(); const s = useStyles(); const dangerText = useDangerTextClass(); - const [streamView, setStreamView] = useState<'collected' | 'events'>('collected'); // Deriving the retained record during render rather than in an effect keeps // the swap out of the first frame of the leave animation. @@ -187,7 +138,6 @@ export function RequestDetailPanel({ collected: loadedCollected, error: loadedEr : { collected: loadedCollected, error: loadedError, record: loadedRecord, recordId: selectedRecordId }; if (shown.recordId !== incoming.recordId) { setShown(incoming); - setStreamView('collected'); } else if (shown.record !== incoming.record || shown.error !== incoming.error) { // Recollecting the same events only rebuilds an equal value, so a reload of // the same record keeps the collected stream and the tab showing it. @@ -195,22 +145,13 @@ export function RequestDetailPanel({ collected: loadedCollected, error: loadedEr } const { collected, error, record, recordId } = shown; - // The two shapes a record comes in. An endpoint served by the onion is - // recorded as its edges and draws the four sections below; a pipelined one is - // recorded as its whole run, and the run's event stream is the whole of it. - const edge = record?.shape === 'edge' ? record : null; - const requestBody = edge ? renderBody(edge.request.body, contentTypeOf(edge.request.headers)) : EMPTY_BODY; - const responseBody = edge?.response.body.type === 'bytes' ? renderBody(edge.response.body.body, contentTypeOf(edge.response.headers)) : EMPTY_BODY; - const streamEvents = useMemo( - () => record?.shape === 'edge' && record.response.body.type === 'stream' ? record.response.body.events : [], - [record], - ); + // A record is a whole run — every stage, both directions — and its event stream is the whole + // of what it holds. + const runEvents = useMemo(() => (record ? renderRunEvents(record.events) : []), [record]); const collectKind = record ? detectCollectKind(record.meta.path) : null; - const renderedEvents = useMemo(() => renderStreamEvents(collectKind, streamEvents), [collectKind, streamEvents]); - const runEvents = useMemo( - () => record?.shape === 'run' ? renderRunEvents(record.events) : [], - [record], - ); + const collectedCopyText = collected?.result === null || collected?.result === undefined + ? undefined + : JSON.stringify(collected.result, null, 2); if (!recordId) return
{t('dashboard.requests.selectPrompt')}
; // This replaces every section rather than sitting in one, so it takes the @@ -222,101 +163,48 @@ export function RequestDetailPanel({ collected: loadedCollected, error: loadedEr const severity = requestSeverity(record.meta.status, record.meta.error); const responseError = errorLabel(record.meta.error); - if (record.shape === 'run') { - return ( - -
- - - {record.meta.path} - {record.meta.status ?? t('dashboard.requests.noStatus')} - {responseError && {responseError}} - } - copyText={record.events || undefined} - /> - - {runEvents.length === 0 ? {t('dashboard.requests.noRunEvents')} : runEvents.map((event, index) => ( -
-
- {event.type || t('dashboard.requests.unlabeled')} - {event.subject && {event.subject}} - {event.parseError && {t('dashboard.requests.jsonParseFailed')}} -
- -
- ))} -
-
-
- ); - } - - const requestHeadersCopy = record.request.headers.map(([name, value]) => `${name}: ${value}`).join('\n'); - const responseHeadersCopy = record.response.headers.map(([name, value]) => `${name}: ${value}`).join('\n'); - const collectedCopyText = collected?.result === null || collected?.result === undefined - ? undefined - : JSON.stringify(collected.result, null, 2); - return ( -
- {record.request.path}} copyText={requestHeadersCopy} /> - -
-
- - - {requestBody.decodeError && {t('dashboard.requests.decodeError', { error: requestBody.decodeError })}} - {requestBody.text ? : {t('dashboard.requests.noRequestBody')}} - -
-
- {record.response.status ?? t('dashboard.requests.noStatus')}{responseError && {responseError}}} copyText={record.response.headers.length ? responseHeadersCopy : undefined} /> - - {record.response.headers.length ? : {t('dashboard.requests.noResponseHeaders')}} - -
setStreamView(data.value as 'collected' | 'events')} size="small"> - {t('dashboard.requests.collected')} - {t('dashboard.requests.events', { count: streamEvents.length })} - - ) : undefined} - copyText={record.response.body.type === 'bytes' && responseBody.text - ? responseBody.copyText - : record.response.body.type === 'stream' && streamView === 'events' - ? streamEventsCopyText(collectKind, streamEvents) - : record.response.body.type === 'stream' && streamView === 'collected' - ? collectedCopyText - : undefined} + title={t('dashboard.requests.run')} + detail={<> + + {record.meta.path} + {record.meta.status ?? t('dashboard.requests.noStatus')} + {responseError && {responseError}} + } + copyText={record.events || undefined} /> - {record.response.body.type === 'none' ? {t('dashboard.requests.noResponseBody')} : null} - {record.response.body.type === 'bytes' && (responseBody.text ? : {t('dashboard.requests.emptyBody')})} - {record.response.body.type === 'stream' && streamView === 'collected' && ( - collectKind === null ? {t('dashboard.requests.noCollector')} - : collected === null ? null - : <> - {collected.error && {collected.error}} - {!collected.error && collected.truncated && {t('dashboard.requests.truncatedStream')}} - {collected.result !== null && } - - )} - {record.response.body.type === 'stream' && streamView === 'events' && renderedEvents.map((event, index) => ( + {runEvents.length === 0 ? {t('dashboard.requests.noRunEvents')} : runEvents.map((event, index) => (
- {/* Not `formatDuration`: its ladder rounds the sub-millisecond - gaps within a burst to `0ms`. */} -
{event.event ?? t('dashboard.requests.unlabeled')}{event.parseError && {t('dashboard.requests.jsonParseFailed')}}+{event.timestamp.toFixed(event.timestamp < 1 ? 3 : 0)}ms
+
+ {event.type || t('dashboard.requests.unlabeled')} + {event.subject && {event.subject}} + {event.parseError && {t('dashboard.requests.jsonParseFailed')}} +
))}
+ {/* The frames are in the run's own stream above; what this adds is the one value they add + up to, which is what a reader compares an answer against. A path no collector reads — + an endpoint that never streams — says so rather than showing an empty section. */} +
+ + + {collectKind === null + ? {t('dashboard.requests.noCollector')} + : collected === null ? {t('dashboard.requests.noRunEvents')} + : <> + {collected.error && {collected.error}} + {!collected.error && collected.truncated && {t('dashboard.requests.truncatedStream')}} + {collected.result !== null && } + } + +
); } diff --git a/apps/web/src/components/requests/header-redact.ts b/apps/web/src/components/requests/header-redact.ts deleted file mode 100644 index 6813f5b9c4..0000000000 --- a/apps/web/src/components/requests/header-redact.ts +++ /dev/null @@ -1,15 +0,0 @@ -const SENSITIVE_HEADERS = new Set([ - 'authorization', - 'cookie', - 'proxy-authorization', - 'set-cookie', - 'x-api-key', - 'x-goog-api-key', -]); - -export const isSensitiveHeader = (name: string) => SENSITIVE_HEADERS.has(name.toLowerCase()); - -export const redactHeaderValue = (value: string): string => { - if (value.length <= 16) return '•'.repeat(value.length); - return `${value.slice(0, 8)}${'•'.repeat(value.length - 16)}${value.slice(-8)}`; -}; diff --git a/apps/web/src/components/requests/run-stream.ts b/apps/web/src/components/requests/run-stream.ts new file mode 100644 index 0000000000..74b0c40978 --- /dev/null +++ b/apps/web/src/components/requests/run-stream.ts @@ -0,0 +1,40 @@ +// The frames a run served, read back out of its own event stream. +// +// A record holds every stage and both directions; the frames the client was served are in it as +// `stream.frame` events, written against the run's object space. So collecting them is reading +// that space — which is what `createRunReader` is — and then handing the frames to the same +// collector the dashboard has always used. +// +// A run may hold more than one stream. What the client read is the one the family's edge teed, +// which is the first opened, so that is the one collected: a sub-request's stream belongs to the +// sub-request and would fold into a different answer. + +import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; + +/** The frames of the run's first stream, in the shape the collector reads. A record with no + * stream — a refusal, a turn answered from a value — has none, and collects to nothing. */ +export const streamEventsOf = (ndjson: string): DumpStreamEvent[] => { + const read = createRunReader(); + const events: DumpStreamEvent[] = []; + let first: number | null = null; + for (const line of ndjson.split('\n')) { + if (line.length === 0) continue; + let event: DumpEvent; + try { + event = JSON.parse(line) as DumpEvent; + } catch { + // A line this cannot parse is a line the collector cannot use either. The event list + // beside it shows the record as stored, failure named, which is where a reader looks. + continue; + } + const said = read(event); + if (event.type !== 'stream.frame') continue; + first ??= event.streamId; + if (event.streamId !== first) continue; + for (const frame of said?.frames ?? []) { + events.push({ frame: frame as DumpStreamEvent['frame'], ts: 0 }); + } + } + return events; +}; diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index 79892a0192..d69fdf9e60 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -955,24 +955,12 @@ const en = { noKeys: 'No API key has request retention enabled', noKeysDescription: 'Enable request dump retention on an API key to start capturing requests.', apiKeysLink: 'API Keys', - request: 'Request', - requestBody: 'Request body', run: 'Run', noRunEvents: 'This run recorded no events.', - response: 'Response', - responseBody: 'Response body', - noRequestBody: 'No request body', - noResponseBody: 'The request did not produce a response body.', - noResponseHeaders: 'No response headers', emptyBody: 'Empty body', truncatedStream: 'This recording was cut short; the response below is what could be reassembled from it.', noStatus: 'No status', - revealValue: 'Reveal value', - hideValue: 'Hide value', - decodeError: 'Could not decode this body ({{error}}); showing raw base64.', - streamView: 'Response body view', collected: 'Collected', - events: 'Events ({{count, number}})', noCollector: 'No protocol-specific collector is available for this path. Use Events to inspect raw frames.', unlabeled: 'unlabeled', jsonParseFailed: 'JSON parse failed', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index 67df712331..9ab6ad50b3 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -908,24 +908,12 @@ const zhHansCN = { noKeys: '没有 API 密钥启用请求保留', noKeysDescription: '在 API 密钥上启用请求转储保留后,即可开始捕获请求。', apiKeysLink: 'API 密钥', - request: '请求', - requestBody: '请求体', run: '运行过程', noRunEvents: '这次运行没有记录到任何事件。', - response: '响应', - responseBody: '响应体', - noRequestBody: '没有请求体', - noResponseBody: '此请求未产生响应体。', - noResponseHeaders: '没有响应 header', emptyBody: '响应体为空', truncatedStream: '这次记录被截断,下面是从中能还原出的部分响应。', noStatus: '无状态码', - revealValue: '显示值', - hideValue: '隐藏值', - decodeError: '无法解码此内容({{error}}),以下显示原始 base64。', - streamView: '响应体视图', collected: '聚合结果', - events: '事件({{count, number}})', noCollector: '此路径没有对应的协议聚合器,请切换到事件视图检查原始帧。', unlabeled: '未命名', jsonParseFailed: 'JSON 解析失败', diff --git a/apps/web/src/routes/dashboard-monitor-requests.tsx b/apps/web/src/routes/dashboard-monitor-requests.tsx index f7caa264f6..10ce818c67 100644 --- a/apps/web/src/routes/dashboard-monitor-requests.tsx +++ b/apps/web/src/routes/dashboard-monitor-requests.tsx @@ -10,6 +10,7 @@ import type { ApiKey } from '../api/types'; import { RequestDetailPanel } from '../components/requests/detail'; import { refreshRequestKeys } from '../components/requests/key-refresh'; import { RequestListPanel } from '../components/requests/list'; +import { streamEventsOf } from '../components/requests/run-stream'; import { collectStream, detectCollectKind, type CollectedStream } from '../components/requests/stream-render'; import { useDumpSubscription } from '../components/requests/use-dump-subscription'; import { DashboardPageHeader } from '../components/ui/dashboard-page-header'; @@ -66,9 +67,9 @@ export async function clientLoader({ request }: Route.ClientLoaderArgs): Promise ]); const record = recordResult?.data ?? null; const collectKind = record ? detectCollectKind(record.meta.path) : null; - // Only an edge-shaped record carries a captured frame log to collect; a run - // records its stream inside its own events and the detail panel reads it there. - const streamEvents = record?.shape === 'edge' && record.response.body.type === 'stream' ? record.response.body.events : []; + // The frames the client was served are in the run's own stream, so the collected view is + // assembled from the record rather than from a second capture beside it. + const streamEvents = record ? streamEventsOf(record.events) : []; const collected = collectKind && streamEvents.length ? await collectStream(collectKind, streamEvents) : null; return { collected, diff --git a/packages/gateway/__tests__/control-plane/dump_test.ts b/packages/gateway/__tests__/control-plane/dump_test.ts index 2bebb7dd7c..fd8e6ca65a 100644 --- a/packages/gateway/__tests__/control-plane/dump_test.ts +++ b/packages/gateway/__tests__/control-plane/dump_test.ts @@ -3,15 +3,17 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; import type { DumpStore } from '../../src/dump/store-contract.ts'; import type { DumpMetadata, DumpRecord, StoredDumpRecord } from '../../src/dump/types.ts'; -import { fakeMeta as baseFakeMeta, fakeRecord as baseFakeRecord, fakeRunRecord, installDumpStubs } from '../dump/test-fixtures.ts'; +import { fakeMeta as baseFakeMeta, fakeRunRecord, installDumpStubs } from '../dump/test-fixtures.ts'; import { requestApp, setupAppTest } from '../test-utils/app.ts'; import { assertEquals, assertExists } from '@floway-dev/test-utils'; const fakeMeta = (id: string, completedAt: number): DumpMetadata => baseFakeMeta({ id, completedAt, startedAt: completedAt - 1 }); +// A record with an empty stream: these cases are about the list and the row, not about what a +// run put in it. const fakeRecord = (id: string, completedAt: number): StoredDumpRecord => - baseFakeRecord({ id, completedAt, startedAt: completedAt - 1 }); + fakeRunRecord([], { id, completedAt, startedAt: completedAt - 1 }); test('GET /api/dump/keys/:keyId/records lists newest-first', async () => { const { repo, apiKey } = await setupAppTest(); @@ -80,14 +82,12 @@ test('GET /api/dump/keys/:keyId/records/:recordId returns the rehydrated record' assertEquals(body.meta.id, '01HZZ0000000000000000000XX'); }); -// One list, both shapes — the metadata is common, so the dashboard's list says -// nothing about which mechanism served a turn — and a detail fetch hands each -// one back as what it is. -test('GET /api/dump/keys/:keyId/records serves a run record beside an edge one', async () => { +// The list is metadata and the detail is the run's own event stream. Both come from the same +// row, which is what lets the dashboard list a turn without opening it. +test('GET /api/dump/keys/:keyId/records serves the run a record holds', async () => { const { repo, apiKey } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const stubs = installDumpStubs(initDumpStore, initDumpBroker); - stubs.seed(apiKey.id, fakeRecord('01HZZ0000000000000000EDGE', 1000)); stubs.seed(apiKey.id, fakeRunRecord( [ { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'request.payload': { model: 'm' } } }, @@ -99,20 +99,14 @@ test('GET /api/dump/keys/:keyId/records serves a run record beside an edge one', const listed = await requestApp(`/api/dump/keys/${apiKey.id}/records`, { headers: { 'x-api-key': apiKey.key } }); assertEquals(listed.status, 200); const { records } = await listed.json() as { records: DumpMetadata[] }; - assertEquals(records.map(meta => meta.id), ['01HZZ00000000000000000RUN', '01HZZ0000000000000000EDGE']); + assertEquals(records.map(meta => meta.id), ['01HZZ00000000000000000RUN']); const run = await requestApp(`/api/dump/keys/${apiKey.id}/records/01HZZ00000000000000000RUN`, { headers: { 'x-api-key': apiKey.key } }); const runBody = await run.json() as DumpRecord; - if (runBody.shape !== 'run') throw new Error(`expected the run shape, got ${runBody.shape}`); // The object space rides with the events that reference it, so the `object` // event carrying the payload arrives before the entry that points at it. assertEquals(runBody.events.trimEnd().split('\n').map(line => (JSON.parse(line) as { type: string }).type), ['object', 'stage.entered', 'stage.leaved']); - - const edge = await requestApp(`/api/dump/keys/${apiKey.id}/records/01HZZ0000000000000000EDGE`, { headers: { 'x-api-key': apiKey.key } }); - const edgeBody = await edge.json() as DumpRecord; - if (edgeBody.shape !== 'edge') throw new Error(`expected the edge shape, got ${edgeBody.shape}`); - assertEquals(edgeBody.request.method, 'POST'); }); test('GET /api/dump/keys/:keyId/records/:recordId 404s on unknown id', async () => { diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts index 5edead4868..bc71c9bd7d 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts @@ -317,10 +317,9 @@ test('/v1/completions non-streaming records usage row, performance neutral row ( assertEquals(dump.meta.model, 'davinci-002'); assertEquals(dump.meta.inputTokens, 7); assertEquals(dump.meta.outputTokens, 2); - // The shape follows the endpoint: a pipelined turn is recorded as its whole run, so what - // is stored is the event stream rather than the two edges. The metadata is common to both, - // which is what lets the dashboard list them together. - assertEquals(runRecordOf(dumpStubs.stored[0]!.record).shape, 'run'); + // A turn is recorded as its whole run, so what is stored is the event stream and the metadata + // beside it — which is what lets the dashboard list a turn without opening it. + assertEquals(eventsOf(runRecordOf(dumpStubs.stored[0]!.record)).length > 0, true); }); // A stream that stops before its terminator did not produce what it said it would, and the diff --git a/packages/gateway/__tests__/data-plane/shared/gateway-ctx_test.ts b/packages/gateway/__tests__/data-plane/shared/gateway-ctx_test.ts index 85f56aae00..961600a7b7 100644 --- a/packages/gateway/__tests__/data-plane/shared/gateway-ctx_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/gateway-ctx_test.ts @@ -56,7 +56,7 @@ describe('createGatewayCtxFromHono', () => { let ctx: ReturnType | undefined; app.get('/test', c => { c.set('apiKey', buildApiKey({ id: 'key-1', upstreamIds: ['up-1', 'up-2'] })); - ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -69,7 +69,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -82,7 +82,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -94,7 +94,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -106,7 +106,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -119,7 +119,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -134,7 +134,7 @@ describe('createGatewayCtxFromHono', () => { let controller: AbortController | undefined; app.get('/test', c => { controller = new AbortController(); - ctx = createGatewayCtxFromHono(c, { wantsStream: true, downstreamAbortController: controller, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: true, downstreamAbortController: controller, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -150,7 +150,7 @@ describe('createGatewayCtxFromHono', () => { let ctx: ReturnType | undefined; const scheduler: BackgroundScheduler = () => {}; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: scheduler }); + ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: scheduler, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -166,21 +166,21 @@ describe('createGatewayCtxFromHono', () => { app.get('/cap-only', c => { // Unrestricted key (apiKey.upstreamIds null) under a capped user. c.set('user', buildUser({ upstreamIds: ['up-a'] })); - collected.capOnly = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }).upstreamIds; + collected.capOnly = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }).upstreamIds; return c.text('ok'); }); app.get('/both', c => { // Per-key whitelist further narrows the user cap and preserves per-key order. c.set('user', buildUser({ upstreamIds: ['up-a', 'up-b'] })); c.set('apiKey', buildApiKey({ upstreamIds: ['up-b', 'up-c'] })); - collected.both = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }).upstreamIds; + collected.both = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }).upstreamIds; return c.text('ok'); }); app.get('/key-only', c => { // Uncapped user with a per-key whitelist falls through to the per-key // list verbatim. c.set('apiKey', buildApiKey({ upstreamIds: ['up-x'] })); - collected.keyOnly = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }).upstreamIds; + collected.keyOnly = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }).upstreamIds; return c.text('ok'); }); await app.request('/cap-only'); diff --git a/packages/gateway/__tests__/dump/run-sink_test.ts b/packages/gateway/__tests__/dump/run-sink_test.ts index 25bc74bf4d..6a86fce5e9 100644 --- a/packages/gateway/__tests__/dump/run-sink_test.ts +++ b/packages/gateway/__tests__/dump/run-sink_test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { installDumpStubs } from './test-fixtures.ts'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; import { openRunDump } from '../../src/dump/run-sink.ts'; -import type { StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; +import type { StoredDumpRecord } from '../../src/dump/types.ts'; import type { ApiKey } from '../../src/repo/types.ts'; import { flushBackground, trackBackground } from '../test-utils/background-tracker.ts'; import { compose, defineStage, move, run, type DumpEvent, type Event } from '@floway-dev/pipeline'; @@ -53,15 +53,14 @@ const shout = defineStage, Pick, Pick, Pick>('shout-it', [shout, answer]); -const runRecordOf = (stored: { record: StoredDumpRecord } | undefined): StoredDumpRunRecord => { +const runRecordOf = (stored: { record: StoredDumpRecord } | undefined): StoredDumpRecord => { if (!stored) throw new Error('expected a stored dump record'); - if (stored.record.shape !== 'run') throw new Error(`expected the run shape, got ${stored.record.shape}`); return stored.record; }; -const ndjson = (record: StoredDumpRunRecord): string => new TextDecoder().decode(record.events); +const ndjson = (record: StoredDumpRecord): string => new TextDecoder().decode(record.events); -const lines = (record: StoredDumpRunRecord): DumpEvent[] => +const lines = (record: StoredDumpRecord): DumpEvent[] => ndjson(record).split('\n').filter(Boolean).map(line => JSON.parse(line) as DumpEvent); test('a run under a key with retention stores its whole event stream as NDJSON', async () => { diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts index 30e5068dd2..24bbec087b 100644 --- a/packages/gateway/__tests__/dump/test-fixtures.ts +++ b/packages/gateway/__tests__/dump/test-fixtures.ts @@ -1,6 +1,6 @@ import type { DumpBroker } from '../../src/dump/broker.ts'; import type { DumpStore } from '../../src/dump/store-contract.ts'; -import type { DumpMetadata, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; +import type { DumpMetadata, StoredDumpRecord } from '../../src/dump/types.ts'; import { encodeRun, toNdjson, type Event } from '@floway-dev/pipeline'; export const fakeMeta = (overrides: Partial = {}): DumpMetadata => ({ @@ -21,38 +21,21 @@ export const fakeMeta = (overrides: Partial = {}): DumpMetadata => ...overrides, }); -export const fakeRecord = (overrides: Partial = {}): StoredDumpEdgeRecord => ({ - shape: 'edge', - meta: fakeMeta(overrides), - request: { method: 'POST', path: '/v1/x', headers: [], body: new Uint8Array() }, - response: { status: 200, headers: [], body: { type: 'none' } }, -}); - // A run record holds one NDJSON stream and no edge halves, so a fixture takes // the events a run emitted and encodes them the way the sink does — object // space, folding and all. -export const fakeRunRecord = (events: readonly Event[], overrides: Partial = {}): StoredDumpRunRecord => ({ - shape: 'run', +export const fakeRunRecord = (events: readonly Event[], overrides: Partial = {}): StoredDumpRecord => ({ meta: fakeMeta(overrides), events: new TextEncoder().encode(toNdjson(encodeRun(events))), }); -// A reader hands back whichever shape was written, so a test asserting on the -// request or response halves says which one it wrote. -export const edgeRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpEdgeRecord => { +export const runRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpRecord => { if (!record) throw new Error('expected a stored dump record'); - if (record.shape !== 'edge') throw new Error(`expected the edge shape, got ${record.shape}`); - return record; -}; - -export const runRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpRunRecord => { - if (!record) throw new Error('expected a stored dump record'); - if (record.shape !== 'run') throw new Error(`expected the run shape, got ${record.shape}`); return record; }; /** The events a run recorded, decoded from the NDJSON one line at a time. */ -export const eventsOf = (record: StoredDumpRunRecord): readonly Record[] => +export const eventsOf = (record: StoredDumpRecord): readonly Record[] => new TextDecoder().decode(record.events).split('\n').filter(line => line.length > 0) .map(line => JSON.parse(line) as Record); @@ -85,19 +68,11 @@ export const installDumpStubs = ( const throws: Partial> = {}; const store: DumpStore = { - async prepareRequestBody(body) { - return { encoding: 'identity', bytes: body, decodedByteLength: body.byteLength }; - }, async put(keyId, record) { if (throws.put) throw throws.put; - // The run shape is already stored-shaped; only the edge shape carries a - // separately prepared request body to rehydrate. - if (record.shape === 'edge' && record.request.body.encoding !== 'identity') { - throw new Error('dump test stub expected identity request body'); - } - const storedRecord: StoredDumpRecord = record.shape === 'run' - ? record - : { ...record, request: { ...record.request, body: record.request.body.bytes } }; + // A record is already stored-shaped: its stream is encoded once the run is over, so there + // is nothing prepared ahead of the write to rehydrate. + const storedRecord: StoredDumpRecord = record; stored.push({ keyId, record: storedRecord }); const list = records.get(keyId) ?? []; list.unshift(storedRecord); diff --git a/packages/gateway/__tests__/dump/wire_test.ts b/packages/gateway/__tests__/dump/wire_test.ts index e4abae2283..9c01550c67 100644 --- a/packages/gateway/__tests__/dump/wire_test.ts +++ b/packages/gateway/__tests__/dump/wire_test.ts @@ -1,11 +1,10 @@ import { expect, test } from 'vitest'; -import type { DumpEdgeRecord, StoredDumpEdgeRecord } from '../../src/dump/types.ts'; +import type { StoredDumpRecord } from '../../src/dump/types.ts'; import { dumpRecordToWire } from '../../src/dump/wire.ts'; import { assertEquals } from '@floway-dev/test-utils'; -const baseStored = (overrides: Partial = {}): StoredDumpEdgeRecord => ({ - shape: 'edge', +const stored = (events: Uint8Array): StoredDumpRecord => ({ meta: { id: 'rec', startedAt: 0, @@ -22,125 +21,23 @@ const baseStored = (overrides: Partial = {}): StoredDumpEd durationMs: 1, error: null, }, - request: { method: 'POST', path: '/v1/x', headers: [], body: new Uint8Array() }, - response: { status: 200, headers: [], body: { type: 'none' } }, - ...overrides, + events, }); -const edgeWire = (record: StoredDumpEdgeRecord): DumpEdgeRecord => { - const wire = dumpRecordToWire(record); - if (wire.shape !== 'edge') throw new Error('expected the edge shape'); - return wire; -}; - -// A textual request content-type with valid UTF-8 bytes serializes as utf8 on -// the wire — the dashboard reads `data` directly as a string. -test('dumpRecordToWire encodes a textual request body as utf8', () => { - const wire = edgeWire(baseStored({ - request: { - method: 'POST', - path: '/v1/messages', - headers: [['content-type', 'application/json']], - body: new TextEncoder().encode('{"k":"v"}'), - }, - })); - assertEquals(wire.request.body.encoding, 'utf8'); - assertEquals(wire.request.body.data, '{"k":"v"}'); -}); - -test('dumpRecordToWire recognizes structured textual suffixes without accepting near matches', () => { - const structured = edgeWire(baseStored({ - request: { - method: 'POST', - path: '/v1/x', - headers: [['content-type', 'Application/Problem+JSON; charset=utf-8']], - body: new TextEncoder().encode('{"error":"bad"}'), - }, - })); - assertEquals(structured.request.body.encoding, 'utf8'); - - const nearMatch = edgeWire(baseStored({ - request: { - method: 'POST', - path: '/v1/x', - headers: [['content-type', 'application/jsonish']], - body: new TextEncoder().encode('{"not":"json"}'), - }, - })); - assertEquals(nearMatch.request.body.encoding, 'base64'); -}); - -// A binary content-type round-trips through base64 so JSON serialization -// preserves every byte; the dashboard decodes base64 client-side. -test('dumpRecordToWire encodes a binary response body as base64', () => { - const png = new Uint8Array([0x89, 0x50, 0x4E, 0x47]); // PNG magic - const wire = edgeWire(baseStored({ - response: { - status: 200, - headers: [['content-type', 'image/png']], - body: { type: 'bytes', body: png }, - }, - })); - if (wire.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(wire.response.body.body.encoding, 'base64'); - // Decoding the base64 back to bytes must reproduce the original sequence. - const binary = atob(wire.response.body.body.data); - assertEquals(Array.from(binary, c => c.charCodeAt(0)), [0x89, 0x50, 0x4E, 0x47]); -}); - -// A content-type that claims to be text but carries bytes that do not decode -// as UTF-8 falls through to base64 so the wire never silently corrupts. -test('dumpRecordToWire falls back to base64 when textual content-type carries non-UTF-8 bytes', () => { - const bytes = new Uint8Array([0xFF, 0xFE, 0xFD]); - const wire = edgeWire(baseStored({ - request: { - method: 'POST', - path: '/v1/x', - headers: [['content-type', 'text/plain']], - body: bytes, - }, - })); - assertEquals(wire.request.body.encoding, 'base64'); - const binary = atob(wire.request.body.data); - assertEquals(Array.from(binary, c => c.charCodeAt(0)), [0xFF, 0xFE, 0xFD]); -}); - -// `stream` and `none` response bodies pass through wire serialization -// unchanged because they carry no raw bytes. -test('dumpRecordToWire passes stream + none response bodies through', () => { - const streamWire = edgeWire(baseStored({ - response: { status: 200, headers: [], body: { type: 'stream', events: [] } }, - })); - assertEquals(streamWire.response.body.type, 'stream'); - - const noneWire = edgeWire(baseStored({ - response: { status: null, headers: [], body: { type: 'none' } }, - })); - assertEquals(noneWire.response.body.type, 'none'); -}); - -// A run record crosses to the wire as the stored bytes decoded, because a line -// of NDJSON is one event and one SSE `data:` payload — what the dashboard reads -// here is what a live observer will read frame by frame. -test('dumpRecordToWire hands a run record its NDJSON verbatim', () => { +// A record crosses to the wire as the stored bytes decoded, because a line of NDJSON is one +// event and one SSE `data:` payload — what the dashboard reads here is what a live observer +// will read frame by frame. +test('dumpRecordToWire hands a record its NDJSON verbatim', () => { const ndjson = '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n' + '{"type":"stage.leaved","stageId":1,"facts":{"response.http.status":200}}\n'; - const wire = dumpRecordToWire({ - shape: 'run', - meta: baseStored().meta, - events: new TextEncoder().encode(ndjson), - }); - if (wire.shape !== 'run') throw new Error('expected the run shape'); + const wire = dumpRecordToWire(stored(new TextEncoder().encode(ndjson))); + assertEquals(wire.events, ndjson); assertEquals(wire.meta.id, 'rec'); }); -// The gateway wrote those bytes itself, so bytes that are not UTF-8 mean a -// corrupted record and the reader says so rather than serving mojibake. -test('dumpRecordToWire refuses a run stream that is not UTF-8', () => { - expect(() => dumpRecordToWire({ - shape: 'run', - meta: baseStored().meta, - events: new Uint8Array([0xFF, 0xFE, 0xFD]), - })).toThrow(); +// The gateway wrote those bytes itself, so bytes that are not UTF-8 mean a corrupted record and +// the reader says so rather than serving mojibake. +test('dumpRecordToWire refuses a stream that is not UTF-8', () => { + expect(() => dumpRecordToWire(stored(new Uint8Array([0xFF, 0xFE, 0xFD])))).toThrow(); }); diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts index d0ca1d14f0..425431017b 100644 --- a/packages/gateway/__tests__/repo/dump-store_test.ts +++ b/packages/gateway/__tests__/repo/dump-store_test.ts @@ -6,7 +6,7 @@ import { expect, test } from 'vitest'; import { createSqliteTestDb, mapRunChangeCount } from './test-sqlite.ts'; import { decodeDumpBodyDescriptor } from '../../src/dump/storage-codec.ts'; -import type { DumpWriteEdgeRecord, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; +import type { StoredDumpRecord } from '../../src/dump/types.ts'; import { FileDumpStore } from '../../src/repo/dump-store.ts'; import { initRepo } from '../../src/repo/index.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; @@ -33,155 +33,17 @@ const openDb = async (): Promise => { return db; }; -const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s); - -const requestBody = utf8('{"hello":"world"}'); - -const baseRecord = (id: string, completedAt: number): DumpWriteEdgeRecord => ({ - shape: 'edge', - meta: { - id, startedAt: completedAt - 1, completedAt, method: 'POST', path: '/v1/x', status: 200, - upstream: null, model: 'm', inputTokens: 1, outputTokens: 2, - requestBytes: 3, responseBytes: 4, durationMs: 1, error: null, - }, - request: { - method: 'POST', path: '/v1/x', - headers: [['content-type', 'application/json']], - body: { encoding: 'identity', bytes: requestBody, decodedByteLength: requestBody.byteLength }, - }, - response: { - status: 200, - headers: [['content-type', 'application/json']], - body: { type: 'bytes', body: utf8('{"id":"abc"}') }, - }, -}); - -// Every fixture above writes the edge shape, and a reader hands back what was -// written — so a test that asserts on the request or response halves says so. -const edgeOf = (record: StoredDumpRecord | null): StoredDumpEdgeRecord => { - assertExists(record); - if (record.shape !== 'edge') throw new Error('expected the edge shape'); - return record; -}; - -test('FileDumpStore prepares request gzip before terminal persistence', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const raw = utf8(`{"content":"${'repeatable '.repeat(4096)}"}`); - const prepared = await store.prepareRequestBody(raw); - const base = baseRecord('01HZZ0000000000000000000P1', Date.UTC(2026, 5, 1, 12, 0, 0)); - const record: DumpWriteEdgeRecord = { - ...base, - meta: { ...base.meta, requestBytes: raw.byteLength }, - request: { - method: 'POST', - path: '/v1/x', - headers: [['content-type', 'application/json']], - body: prepared, - }, - }; - - assertEquals(prepared.encoding, 'gzip'); - assertEquals(prepared.decodedByteLength, raw.byteLength); - assertEquals(prepared.bytes.byteLength < raw.byteLength, true); - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', record.meta.id)); - assertEquals(Array.from(fetched.request.body), Array.from(raw)); -}); - -test('FileDumpStore round-trips a JSON record through gzip', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record = baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); - - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A1')); - assertEquals(fetched.meta.id, record.meta.id); - assertEquals(new TextDecoder().decode(fetched.request.body), '{"hello":"world"}'); - if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(new TextDecoder().decode(fetched.response.body.body), '{"id":"abc"}'); -}); - -test('FileDumpStore preserves the original content-type header on binary bodies', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const pngMagic = new Uint8Array([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]); - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ0000000000000000000PNG', Date.UTC(2026, 5, 1, 12, 0, 0)), - response: { - status: 200, - headers: [['content-type', 'image/png']], - body: { type: 'bytes', body: pngMagic }, - }, - }; - - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000PNG')); - // The header pair must survive verbatim — no `;base64` suffix tacked on. - assertEquals(fetched.response.headers.find(([k]) => k === 'content-type')?.[1], 'image/png'); - if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(Array.from(fetched.response.body.body), Array.from(pngMagic)); -}); - -test('FileDumpStore preserves the bytes discriminator on an empty-body response', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - // 204-style: real upstream response with status + headers but a zero-length - // body. Persistence drops the body file (nothing to gzip), but headers are - // still written — the read path must surface this as `bytes`, not `none`. - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ0000000000000000000E1', Date.UTC(2026, 5, 1, 12, 0, 0)), - response: { - status: 204, - headers: [['content-type', 'application/json']], - body: { type: 'bytes', body: new Uint8Array() }, - }, - }; - - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000E1')); - if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(fetched.response.body.body.byteLength, 0); - assertEquals(fetched.response.headers.find(([k]) => k === 'content-type')?.[1], 'application/json'); -}); - -test('FileDumpStore round-trips an SSE record as a stream events array', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0)), - response: { - status: 200, - headers: [['content-type', 'text/event-stream']], - body: { - type: 'stream', - events: [ - { frame: { type: 'event', event: { type: 'message_start' } }, ts: 10 }, - { frame: { type: 'done' }, ts: 20 }, - ], - }, - }, - }; - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A2')); - if (fetched.response.body.type !== 'stream') throw new Error('expected stream'); - assertEquals(fetched.response.body.events.length, 2); - assertEquals(fetched.response.body.events[0]!.frame.type, 'event'); -}); - -// A pipelined turn stores the run rather than its edges: one NDJSON body file -// under the same contract, and a row that carries the same metadata. -const runRecord = (id: string, completedAt: number): StoredDumpRunRecord => { +// A turn is stored as its run: one NDJSON body file under the file-store contract, and a row +// carrying the metadata a list renders without opening it. +const runRecord = (id: string, completedAt: number): StoredDumpRecord => { const entered: Facts = { 'request.http.path': '/v1/embeddings' }; const left: Facts = { ...entered, 'response.http.status': 200 }; return { - shape: 'run', - meta: baseRecord(id, completedAt).meta, + meta: { + id, startedAt: completedAt - 1, completedAt, method: 'POST', path: '/v1/x', status: 200, + upstream: null, model: 'm', inputTokens: 1, outputTokens: 2, + requestBytes: 3, responseBytes: 4, durationMs: 1, error: null, + }, events: new TextEncoder().encode(toNdjson(encodeRun([ { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: entered }, { type: 'stage.entered', stageId: 2, name: 'dial', parentStageId: 1, facts: entered }, @@ -200,11 +62,10 @@ test('FileDumpStore round-trips a run record as its NDJSON event stream', async await store.put('key_x', record); const fetched = await store.get('key_x', record.meta.id); assertExists(fetched); - if (fetched.shape !== 'run') throw new Error(`expected the run shape, got ${fetched.shape}`); assertEquals(new TextDecoder().decode(fetched.events), new TextDecoder().decode(record.events)); - // The stream is a body file like the other two: gzipped, named for what it - // holds, and pointed at by the row's descriptor. + // The stream is a body file: gzipped, named for what it holds, and pointed at by the row's + // response descriptor — which is also what says the row has a run to read. const row = await db.prepare('SELECT request_body_descriptor, response_body_descriptor FROM dump_records WHERE key_id = ? AND id = ?') .bind('key_x', record.meta.id) .first<{ request_body_descriptor: string | null; response_body_descriptor: string }>(); @@ -215,17 +76,16 @@ test('FileDumpStore round-trips a run record as its NDJSON event stream', async assertEquals(descriptor.key.endsWith('.run.gz'), true); }); -test('FileDumpStore lists a run record alongside an edge-shaped one', async () => { +test('FileDumpStore lists what a row renders without opening the stream', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); const base = Date.UTC(2026, 5, 1, 12, 0, 0); - await store.put('key_x', baseRecord('01HZZ00000000000000000EDG1', base)); + await store.put('key_x', runRecord('01HZZ00000000000000000RUN1', base)); await store.put('key_x', runRecord('01HZZ00000000000000000RUN2', base + 1)); const listed = await store.list('key_x', { limit: 10 }); - assertEquals(listed.map(meta => meta.id), ['01HZZ00000000000000000RUN2', '01HZZ00000000000000000EDG1']); - // One metadata shape for both: the list says nothing about how a turn was - // served, and every column a row renders is filled either way. + assertEquals(listed.map(meta => meta.id), ['01HZZ00000000000000000RUN2', '01HZZ00000000000000000RUN1']); + // Every column a row renders is filled from the metadata, so a list costs no file read. assertEquals(listed.map(meta => meta.model), ['m', 'm']); assertEquals(listed.map(meta => meta.status), [200, 200]); }); @@ -263,7 +123,7 @@ test('FileDumpStore retires an expired run record and collects its stream file', test('FileDumpStore rejects malformed metadata with its row identity', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); - const record = baseRecord('01HZZ000000000000000000BADM', Date.UTC(2026, 5, 1, 12)); + const record = runRecord('01HZZ000000000000000000BADM', Date.UTC(2026, 5, 1, 12)); await store.put('key_x', record); await db.prepare('UPDATE dump_records SET meta_json = ? WHERE key_id = ? AND id = ?') .bind(JSON.stringify({ ...record.meta, upstream: undefined, status: '200' }), 'key_x', record.meta.id) @@ -273,59 +133,33 @@ test('FileDumpStore rejects malformed metadata with its row identity', async () .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000BADM metadata.*status/su); }); -test('FileDumpStore rejects malformed header pairs with their record and side', async () => { +test('FileDumpStore rejects a malformed body descriptor before file access', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); - const record = baseRecord('01HZZ000000000000000000BADH', Date.UTC(2026, 5, 1, 12)); + const record = runRecord('01HZZ000000000000000000BADD', Date.UTC(2026, 5, 1, 12)); await store.put('key_x', record); - await db.prepare('UPDATE dump_records SET request_headers_json = ? WHERE key_id = ? AND id = ?') - .bind(JSON.stringify([['x-bad', 1]]), 'key_x', record.meta.id) + await db.prepare('UPDATE dump_records SET response_body_descriptor = ? WHERE key_id = ? AND id = ?') + .bind(JSON.stringify({ key: 'dumps/v1/key_x/run.gz', type: 'chunks' }), 'key_x', record.meta.id) .run(); await expect(store.get('key_x', record.meta.id)) - .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000BADH request headers/u); + .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000BADD response body descriptor.*type/su); }); -test('FileDumpStore rejects malformed body descriptors before file access', async () => { +// A row whose descriptor is well-formed but names something other than a run has no stream to +// read, and the migration that deleted the shape which wrote those leaves none — so it is +// corruption, and the reader says so rather than handing back half a record. +test('FileDumpStore refuses a row that points at no run stream', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); - const record = baseRecord('01HZZ000000000000000000BADD', Date.UTC(2026, 5, 1, 12)); + const record = runRecord('01HZZ000000000000000000NORUN', Date.UTC(2026, 5, 1, 12)); await store.put('key_x', record); - await db.prepare('UPDATE dump_records SET request_body_descriptor = ? WHERE key_id = ? AND id = ?') - .bind(JSON.stringify({ key: 'dumps/v1/key_x/request.gz', type: 'chunks' }), 'key_x', record.meta.id) + await db.prepare('UPDATE dump_records SET response_body_descriptor = ? WHERE key_id = ? AND id = ?') + .bind(JSON.stringify({ key: 'dumps/v1/key_x/run.gz', type: 'bytes' }), 'key_x', record.meta.id) .run(); await expect(store.get('key_x', record.meta.id)) - .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000BADD request body descriptor.*type/su); -}); - -test('FileDumpStore rejects malformed stream events with record and file context', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ000000000000000000BADE', Date.UTC(2026, 5, 1, 12)), - response: { - status: 200, - headers: [['content-type', 'text/event-stream']], - body: { type: 'stream', events: [{ frame: { type: 'done' }, ts: 1 }] }, - }, - }; - await store.put('key_x', record); - const row = await db.prepare('SELECT response_body_descriptor FROM dump_records WHERE key_id = ? AND id = ?') - .bind('key_x', record.meta.id) - .first<{ response_body_descriptor: string }>(); - if (row === null) throw new Error('expected stored dump row'); - const descriptor = decodeDumpBodyDescriptor(row.response_body_descriptor, 'test response descriptor'); - const malformedEvents = utf8(JSON.stringify([{ frame: { type: 'done' }, ts: 'late' }])); - const compressed = new Uint8Array(await new Response( - new Blob([malformedEvents as BlobPart]).stream().pipeThrough(new CompressionStream('gzip')), - ).arrayBuffer()); - await files.put(descriptor.key, compressed); - - await expect(store.get('key_x', record.meta.id)).rejects.toThrow( - new RegExp(`Invalid dump record ${record.meta.id} response events at key=${descriptor.key}.*ts`, 'su'), - ); + .rejects.toThrow(/dump record 01HZZ000000000000000000NORUN has no run stream to read/u); }); test('FileDumpStore.list paginates newest-first with the (createdAt, id) cursor', async () => { @@ -334,7 +168,7 @@ test('FileDumpStore.list paginates newest-first with the (createdAt, id) cursor' const store = new FileDumpStore(db, files); const base = Date.UTC(2026, 5, 1, 12, 0, 0); for (let i = 0; i < 5; i++) { - await store.put('key_x', baseRecord(`01HZZ000000000000000000A0${i}`, base + i)); + await store.put('key_x', runRecord(`01HZZ000000000000000000A0${i}`, base + i)); } const first = await store.list('key_x', { limit: 2 }); assertEquals(first.map(m => m.id), ['01HZZ000000000000000000A04', '01HZZ000000000000000000A03']); @@ -351,8 +185,8 @@ test('FileDumpStore applies retention immediately and retires exact expired file const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12, 0, 0); // Old bucket 9:xx, current bucket 12:xx. - await store.put('key_x', baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); - await store.put('key_x', baseRecord('01HZZ0000000000000000000A2', now)); + await store.put('key_x', runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); + await store.put('key_x', runRecord('01HZZ0000000000000000000A2', now)); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 2 * 3600 }); const originalNow = Date.now; Date.now = () => now + 1; @@ -381,7 +215,7 @@ test('growing dump retention can reveal a row not yet physically deleted', async const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12); - await store.put('key_x', baseRecord('01HZZ0000000000000000000A4', now - 3 * 3600_000)); + await store.put('key_x', runRecord('01HZZ0000000000000000000A4', now - 3 * 3600_000)); const originalNow = Date.now; Date.now = () => now; try { @@ -401,8 +235,8 @@ test('FileDumpStore retires every dump record when retention is disabled and col const files = new MemoryFileStore(); initFileStore(files); const store = new FileDumpStore(db, files); - await store.put('key_x', baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); - await store.put('key_x', baseRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0))); + await store.put('key_x', runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); + await store.put('key_x', runRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: null }); assertEquals((await store.list('key_x', { limit: 10 })).length, 0); assertEquals(await store.deleteExpiredBatch('key_x', Date.now(), 100), 2); @@ -416,8 +250,8 @@ test('FileDumpStore counts returned dump rows instead of trigger-amplified chang const repo = new SqlRepo(db); const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); - await store.put('key_x', baseRecord('01HZZ0000000000000000000C1', Date.UTC(2026, 5, 1, 9))); - await store.put('key_x', baseRecord('01HZZ0000000000000000000C2', Date.UTC(2026, 5, 1, 10))); + await store.put('key_x', runRecord('01HZZ0000000000000000000C1', Date.UTC(2026, 5, 1, 9))); + await store.put('key_x', runRecord('01HZZ0000000000000000000C2', Date.UTC(2026, 5, 1, 10))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: null }); const d1LikeStore = new FileDumpStore(mapRunChangeCount(db, changes => changes * 3), files); @@ -431,8 +265,8 @@ test('FileDumpStore counts returned active dump rows instead of trigger-amplifie const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12); - await store.put('key_x', baseRecord('01HZZ0000000000000000000C3', Date.UTC(2026, 5, 1, 9))); - await store.put('key_x', baseRecord('01HZZ0000000000000000000C4', Date.UTC(2026, 5, 1, 10))); + await store.put('key_x', runRecord('01HZZ0000000000000000000C3', Date.UTC(2026, 5, 1, 9))); + await store.put('key_x', runRecord('01HZZ0000000000000000000C4', Date.UTC(2026, 5, 1, 10))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 3600 }); const d1LikeStore = new FileDumpStore(mapRunChangeCount(db, changes => changes * 3), files); @@ -447,16 +281,18 @@ test('a record-ID race leaves only the losing write\'s uniquely keyed files coll const files = new MemoryFileStore(); initFileStore(files); const store = new FileDumpStore(db, files); - const record = baseRecord('01HZZ0000000000000000000A3', Date.now()); + const record = runRecord('01HZZ0000000000000000000A3', Date.now()); await store.put('key_x', record); + // One file per record now — the run's own stream — so the winner owns one and the loser + // staged one under a key of its own. await expect(store.put('key_x', record)).rejects.toThrow(); - expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(2); - expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'staged'").first<{ count: number }>())?.count).toBe(2); + expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(1); + expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'staged'").first<{ count: number }>())?.count).toBe(1); await collectSpilledFiles(Date.now() + 60 * 60 * 1000 + 1); expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'staged'").first<{ count: number }>())?.count).toBe(0); - expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(2); + expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(1); assertExists(await store.get('key_x', record.meta.id)); }); @@ -503,13 +339,12 @@ test('FileDumpStore: put + get round-trips through real-filesystem IO', async () try { const db = await openDb(); const store = new FileDumpStore(db, new TmpDirFileStore(join(root, 'files'))); - const record = baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); + const record = runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A1')); - assertEquals(new TextDecoder().decode(fetched.request.body), '{"hello":"world"}'); - if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(new TextDecoder().decode(fetched.response.body.body), '{"id":"abc"}'); + const fetched = await store.get('key_x', '01HZZ0000000000000000000A1'); + assertExists(fetched); + assertEquals(new TextDecoder().decode(fetched.events), new TextDecoder().decode(record.events)); } finally { await rm(root, { recursive: true, force: true }); } diff --git a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts index 3719bd4c72..3dbeb4dad2 100644 --- a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts +++ b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts @@ -38,7 +38,6 @@ const responseItem = (id: string, refreshedAt: number, apiKeyId = 'key-a'): Stor }); const dumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ - shape: 'edge', meta: { id, startedAt: completedAt - 1, @@ -50,18 +49,14 @@ const dumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ model: 'gpt-test', inputTokens: null, outputTokens: null, - requestBytes: 0, - responseBytes: 0, + requestBytes: 1, + responseBytes: 1, durationMs: 1, error: null, }, - request: { - method: 'POST', - path: '/v1/responses', - headers: [], - body: { encoding: 'identity', bytes: new Uint8Array(), decodedByteLength: 0 }, - }, - response: { status: 200, headers: [], body: { type: 'none' } }, + // One line is enough: what these exercise is the row, its file and the sweep that retires + // both, not what a run put in the stream. + events: new TextEncoder().encode('{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n'), }); test('one fair driver drains bounded OpenAI Responses and dump backlogs', async () => { diff --git a/packages/gateway/__tests__/scheduled_test.ts b/packages/gateway/__tests__/scheduled_test.ts index 5bb13c1a15..b13be54dd8 100644 --- a/packages/gateway/__tests__/scheduled_test.ts +++ b/packages/gateway/__tests__/scheduled_test.ts @@ -30,7 +30,6 @@ const apiKey = (id: string, now: number, secretDigit: number): ApiKey => ({ }); const fileBackedDumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ - shape: 'edge', meta: { id, startedAt: completedAt - 1, @@ -47,13 +46,9 @@ const fileBackedDumpRecord = (id: string, completedAt: number): DumpWriteRecord durationMs: 1, error: null, }, - request: { - method: 'POST', - path: '/v1/responses', - headers: [], - body: { encoding: 'identity', bytes: new Uint8Array([1]), decodedByteLength: 1 }, - }, - response: { status: 200, headers: [], body: { type: 'bytes', body: new Uint8Array([2]) } }, + // One line is enough: what these exercise is the row, its file and the sweep that retires + // both, not what a run put in the stream. + events: new TextEncoder().encode('{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n'), }); test('scheduled maintenance isolates the shared expiration driver from later collectors', async () => { @@ -115,14 +110,15 @@ test('one maintenance tick collects every file retired by its four dump units', } const { results: ownedFiles } = await db.prepare('SELECT file_key FROM spilled_files ORDER BY file_key') .all<{ file_key: string }>(); - expect(ownedFiles).toHaveLength(400); + // One file per record — the run's own stream. + expect(ownedFiles).toHaveLength(200); await runScheduledMaintenance(); expect(await db.prepare('SELECT COUNT(*) AS count FROM dump_records').first<{ count: number }>()).toEqual({ count: 0 }); expect(await db.prepare('SELECT COUNT(*) AS count FROM spilled_files').first<{ count: number }>()).toEqual({ count: 0 }); expect(await Promise.all(ownedFiles.map(row => files.get(row.file_key)))) - .toEqual(Array.from({ length: 400 }, () => null)); + .toEqual(Array.from({ length: 200 }, () => null)); }); test('scheduled maintenance lease keeps overlapping ticks within one budget', async () => { diff --git a/packages/gateway/__tests__/vitest.setup.ts b/packages/gateway/__tests__/vitest.setup.ts index bda8bffac7..53eecd00de 100644 --- a/packages/gateway/__tests__/vitest.setup.ts +++ b/packages/gateway/__tests__/vitest.setup.ts @@ -30,7 +30,6 @@ initBackgroundSchedulerResolver(_c => trackBackground); // independent of that subsystem. Dump-specific tests install real or recording // implementations. const noopStore: DumpStore = { - async prepareRequestBody(body) { return { encoding: 'identity', bytes: body, decodedByteLength: body.byteLength }; }, async put(): Promise { /* noop */ }, async list(): Promise { return []; }, async get(_keyId: string, _id: DumpRecordId): Promise { return null; }, diff --git a/packages/gateway/migrations/0084_drop_edge_dump_records.sql b/packages/gateway/migrations/0084_drop_edge_dump_records.sql new file mode 100644 index 0000000000..492687cd2f --- /dev/null +++ b/packages/gateway/migrations/0084_drop_edge_dump_records.sql @@ -0,0 +1,20 @@ +-- Delete the dump records written as a turn's two edges. +-- +-- Every route opens its run through the prologue now, so nothing produces that shape any more: +-- a record is the whole run, every stage and both directions, as one NDJSON event stream. What +-- is left of the old shape is stored rows, and they are deleted rather than kept readable. An +-- edge record holds what the client sent and what it got back and nothing about what happened +-- between, so there is no run to rebuild it into — and a dump record is diagnostic and already +-- expires under its key's own `dump_retention_seconds`, so what this brings forward is a +-- deletion that was coming anyway. +-- +-- Which rows: the reader told the two shapes apart by the response body descriptor's own type, +-- because a run is one more body file under the same contract and says so. Anything that is not +-- a run is an edge record, including a row that stored no response body at all. +-- +-- The files go with them. `dump_records_retire_spilled_files` fires on delete and marks both +-- descriptors' keys retired with `collect_after = 0`, so the collector picks them up on its next +-- pass rather than leaving orphans in the file store. +DELETE FROM dump_records +WHERE response_body_descriptor IS NULL + OR json_extract(response_body_descriptor, '$.type') IS NOT 'run'; diff --git a/packages/gateway/src/data-plane/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/shared/gateway-ctx.ts index fb8af7900f..5d115b4894 100644 --- a/packages/gateway/src/data-plane/shared/gateway-ctx.ts +++ b/packages/gateway/src/data-plane/shared/gateway-ctx.ts @@ -1,5 +1,4 @@ import type { RequestBody } from './request-body.ts'; -import { openDumpAccumulator } from '../../dump/accumulator.ts'; import type { TurnDump } from '../../dump/turn-dump.ts'; import { apiKeyFromContext, type AuthedContext, effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; @@ -46,17 +45,17 @@ export interface GatewayCtx { export interface CreateGatewayCtxOptions { wantsStream: boolean; - // What this turn is recorded as. The shape follows the endpoint: a pipelined one hands in - // its run recording, and everything else lets the factory open the edge accumulator. - // Absent and null differ — absent means "open the usual one", null means "record nothing". - dump?: TurnDump | null; + // What this turn is recorded as, opened by whoever opened the run — every entry does, through + // the prologue. `null` is a key with no retention configured, which is what makes recording + // conditional: there is nothing to hand the runner rather than a sink that discards. + dump: TurnDump | null; // WebSocket-style call sites own the AbortController (so the upgrade // handler can cancel mid-stream); HTTP call sites let the factory mint one // when wantsStream is true. downstreamAbortController?: AbortController; // Already-buffered inbound request body bytes. HTTP handlers read them - // once via `readRequestBody` and pass them in so the dump accumulator's - // snapshot reflects the exact bytes the handler parsed. WebSocket + // once via `readRequestBody` and pass them in so the record holds the exact + // bytes the handler parsed. WebSocket // upgrades carry no HTTP body — the WS OpenAI Responses path passes the // per-turn JSON message bytes here so the dump captures the turn's // input verbatim. @@ -85,7 +84,7 @@ export const createGatewayCtxFromHono = (c: AuthedContext, opts: CreateGatewayCt const controller = opts.downstreamAbortController ?? (opts.wantsStream ? new AbortController() : undefined); const apiKey = apiKeyFromContext(c); const upstreamIds = effectiveUpstreamIdsFromContext(c); - const dump = 'dump' in opts ? opts.dump ?? null : openDumpAccumulator(c, opts.method ?? c.req.method, apiKey, opts.requestBody, opts.backgroundScheduler); + const dump = opts.dump; if (opts.model !== undefined) dump?.requestedModel(opts.model); return { apiKeyId: apiKey.id, @@ -101,8 +100,8 @@ export const createGatewayCtxFromHono = (c: AuthedContext, opts: CreateGatewayCt }; }; -// Run the dump-accumulator's finalize tee on the outgoing Response. Every -// inbound HTTP wrapper returns its response through this seam so the dump -// pipeline applies uniformly across happy-path, error, and passthrough paths. +// Run the recording's finalize tee on the outgoing Response. Every inbound HTTP +// wrapper returns its response through this seam so the dump applies uniformly +// across happy-path, error, and passthrough paths. export const finalizeGatewayResponse = (ctx: GatewayCtx, response: Response): Response => ctx.dump?.finalize(response) ?? response; diff --git a/packages/gateway/src/dump/accumulator.ts b/packages/gateway/src/dump/accumulator.ts deleted file mode 100644 index deebb7e391..0000000000 --- a/packages/gateway/src/dump/accumulator.ts +++ /dev/null @@ -1,271 +0,0 @@ -// Per-request dump pipeline for an endpoint served by the onion: the record is -// the turn's two **edges** — what the client sent, what the client got back. -// Opens the dump session (request snapshot + opt-in decision) and exposes the -// mid-flight hooks the respond layer calls to record outcomes and frames. When -// the api key has no retention configured, opening returns null and the data -// plane pays no per-request cost. -// -// A pipelined endpoint records the whole run instead; `run-sink.ts` is that -// half, and both fill the same `DumpMetadata` through `DumpAttribution`. - -import type { Context } from 'hono'; - -import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; -import { getDumpBroker, getDumpStore } from './registry.ts'; -import type { StreamRecording } from './turn-dump.ts'; -import type { - DumpMetadata, - DumpStreamEvent, - DumpWriteRecord, - PreparedDumpRequestBody, - StoredDumpResponseBody, -} from './types.ts'; -import type { RequestBody } from '../data-plane/shared/request-body.ts'; -import type { ApiKey, TokenUsage } from '../repo/types.ts'; -import { ulid } from '../shared/ulid.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; -import { isEventStreamMediaType, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { TelemetryModelIdentity } from '@floway-dev/provider'; - -// Frozen at ctx construction so `finalize` never has to re-read a stream -// the handler already consumed. -interface RequestSnapshot { - readonly method: string; - readonly path: string; - readonly headers: ReadonlyArray; - readonly bodyByteLength: number; - readonly streamError: string | null; -} - -interface ResponseSnapshot { - readonly status: number; - readonly headers: ReadonlyArray; - readonly isStream: boolean; - readonly bytes: Uint8Array; - readonly payloadBytes: number; - readonly streamError: string | null; -} - -const headerPairs = (headers: Headers): Array<[string, string]> => { - const pairs: Array<[string, string]> = []; - headers.forEach((value, name) => { pairs.push([name, value]); }); - return pairs; -}; - -export class DumpAccumulator { - private readonly attribution = new DumpAttribution(); - private readonly events: DumpStreamEvent[] = []; - private sentPayloadBytes = 0; - private readonly preparedRequestBody: Promise; - - constructor( - private readonly apiKey: ApiKey, - private readonly requestSnapshot: RequestSnapshot, - requestBody: Uint8Array, - private readonly startedAt: number, - private readonly backgroundScheduler: BackgroundScheduler, - ) { - this.preparedRequestBody = getDumpStore().prepareRequestBody(requestBody); - // Preparation starts eagerly and is awaited at terminal persistence. Mark - // a rejection handled immediately so a long upstream wait cannot surface - // it as an unhandled promise before `write()` records the dump failure. - void this.preparedRequestBody.catch(() => {}); - } - - // --- mid-flight hooks (called from per-protocol respond layer) --- - - requestedModel(model: string): void { - this.attribution.requestedModel(model); - } - - error(kind: 'upstream' | 'gateway', upstream?: string): void { - this.attribution.error(kind, upstream); - } - - failed(reason: unknown): void { - this.attribution.failed(reason); - } - - // Records one protocol frame. Stored as the canonical ProtocolFrame so - // neither serialization nor parsing happens on this path; the dashboard - // derives the SSE wire view on demand via the per-protocol - // frame-to-SSE encoder + reducer. - frame(frame: ProtocolFrame): void { - this.events.push({ frame, ts: Date.now() - this.startedAt }); - } - - /** This shape has one frame log and no way to name anything in it, so every stream lands in - * the same place, there is no terminator to write, and no fact can point at one. A turn - * that opened two would have them interleaved — which the endpoints on this shape do not do, - * and which the run shape is what fixes. */ - openStream(): StreamRecording { - return { frame: frame => { this.frame(frame); }, end: () => {}, fact: null }; - } - - recordSentPayloadBytes(byteLength: number): void { - this.sentPayloadBytes += byteLength; - } - - success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { - this.attribution.success(identity, usage); - } - - // --- response-side: handler exit --- - - // Schedules the dump-record write at the turn's terminal point. Two input - // shapes: - // - // • `(status, headers)` — no HTTP Response object to tee. The WebSocket - // OpenAI Responses path uses this: its "response" is the stream of frames - // already captured via `frame()`, while the send seam records their - // actual UTF-8 payload bytes via `recordSentPayloadBytes()`. - // • `(response)` — tees the response body so the client gets bytes - // flowing while a background reader accumulates the other half. The - // returned Response streams the client-side bytes; status, statusText, - // and headers pass through verbatim so the tee is invisible to the - // client. A null body falls through to the bare form. - // - // The background drain → record assembly → store put → broker publish is - // scheduled through the runtime's BackgroundScheduler so dump write - // failures cannot turn a successful upstream call into a 502. - finalize(status: number, headers: ReadonlyArray): void; - finalize(response: Response): Response; - finalize(...args: [number, ReadonlyArray] | [Response]): void | Response { - if (args.length === 2) { - const [status, headers] = args; - this.backgroundScheduler(this.write({ - status, - headers: headers.map(([k, v]) => [k, v]), - isStream: this.events.length > 0, - bytes: new Uint8Array(), - payloadBytes: this.sentPayloadBytes, - streamError: null, - })); - return; - } - - const [response] = args; - const responseStatus = response.status; - const responseHeaders = headerPairs(response.headers); - - if (response.body === null) { - this.finalize(responseStatus, responseHeaders); - return response; - } - - const isStream = isEventStreamMediaType(response.headers.get('content-type')); - const [forClient, forCapture] = response.body.tee(); - this.backgroundScheduler((async () => { - const reader = forCapture.getReader(); - const chunks: Uint8Array[] = []; - let total = 0; - let streamError: string | null = null; - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - chunks.push(value); - total += value.byteLength; - } - } catch (err) { - streamError = oneLineError(err); - } - const bytes = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } - await this.write({ - status: responseStatus, - headers: responseHeaders, - isStream, - bytes, - payloadBytes: bytes.byteLength, - streamError, - }); - })()); - - return new Response(forClient, { - status: response.status, - statusText: response.statusText, - headers: response.headers, - }); - } - - // --- private: persist --- - - private async write(response: ResponseSnapshot): Promise { - // ULID-from-completedAt keeps ids increasing with row creation time; the - // random tail provides the deterministic tie-breaker for one millisecond. - const completedAt = Date.now(); - const recordId = ulid(completedAt); - - // Prefer the accumulator's frame log so dumps reflect the gateway's - // frame sequence regardless of negotiated wire shape; a turn with no - // frames falls back to captured bytes. - const responseBody: StoredDumpResponseBody = this.events.length > 0 - ? { type: 'stream', events: this.events } - : response.bytes.byteLength > 0 || response.streamError !== null - ? response.isStream - ? { type: 'stream', events: [] } - : { type: 'bytes', body: response.bytes } - : { type: 'none' }; - - const meta: DumpMetadata = await this.attribution.metadata({ - id: recordId, - startedAt: this.startedAt, - completedAt, - method: this.requestSnapshot.method, - path: this.requestSnapshot.path, - status: response.status, - requestBytes: this.requestSnapshot.bodyByteLength, - responseBytes: response.payloadBytes, - // Precedence: an explicit error stamp from the respond path wins — the - // assembler applies this only when there is none. - fallbackError: streamReadError(this.requestSnapshot.streamError, response.streamError), - }); - - // Commit the row before publishing so subscribers fetching detail off the meta frame find it. - try { - const record: DumpWriteRecord = { - shape: 'edge', - meta, - request: { - method: this.requestSnapshot.method, - path: this.requestSnapshot.path, - headers: this.requestSnapshot.headers.map(([k, v]) => [k, v]), - body: await this.preparedRequestBody, - }, - response: { - status: response.status, - headers: response.headers.map(([k, v]) => [k, v]), - body: responseBody, - }, - }; - await getDumpStore().put(this.apiKey.id, record); - await getDumpBroker().publish(this.apiKey.id, meta); - } catch (err) { - console.error(`[dump] write failed for key=${this.apiKey.id} record=${recordId}`, oneLineError(err)); - } - } -} - -// Returns null when the api key opts out of dumps; callers then skip all -// per-request dump work. `method` is passed explicitly rather than read -// off the request so the WebSocket OpenAI Responses path can record each turn -// as `WS /v1/responses` rather than the upgrade's `GET`. -export const openDumpAccumulator = ( - c: Context, - method: string, - apiKey: ApiKey, - requestBody: RequestBody, - backgroundScheduler: BackgroundScheduler, -): DumpAccumulator | null => { - if (apiKey.dumpRetentionSeconds === null) return null; - const requestSnapshot: RequestSnapshot = { - method, - path: c.req.path, - headers: headerPairs(c.req.raw.headers), - bodyByteLength: requestBody.bytes.byteLength, - streamError: requestBody.streamError, - }; - return new DumpAccumulator(apiKey, requestSnapshot, requestBody.bytes, Date.now(), backgroundScheduler); -}; diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 2e9c36489f..635432adc3 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -184,7 +184,6 @@ export class RunDump { // Commit the row before publishing so subscribers fetching detail off the meta frame find it. try { await getDumpStore().put(this.apiKey.id, { - shape: 'run', meta, events: new TextEncoder().encode(toNdjson(this.events)), }); diff --git a/packages/gateway/src/dump/store-contract.ts b/packages/gateway/src/dump/store-contract.ts index 8f6f922419..ca4cb62ac9 100644 --- a/packages/gateway/src/dump/store-contract.ts +++ b/packages/gateway/src/dump/store-contract.ts @@ -1,4 +1,4 @@ -import type { DumpMetadata, DumpRecordId, DumpWriteRecord, PreparedDumpRequestBody, StoredDumpRecord } from './types.ts'; +import type { DumpMetadata, DumpRecordId, DumpWriteRecord, StoredDumpRecord } from './types.ts'; // Per-API-key request dump storage contract: metadata in SQL, bodies in the // FileStore. Request bytes are prepared before the terminal write; reads @@ -10,10 +10,6 @@ export interface DumpListOptions { } export interface DumpStore { - // Starts body preparation while the request is in flight. Implementations - // may return identity bytes, but persistent stores compress here so the - // accumulator can release the original request buffer before terminal IO. - prepareRequestBody(body: Uint8Array): Promise; // Write body files BEFORE the metadata row so a partial failure leaves // orphan files (sweep-collectable), not orphan rows (broken records). diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index 55ab15fddf..a65bca7bbc 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -59,116 +59,23 @@ export type DumpMetadata = z.infer; // agnostic — the dashboard dispatches the right per-protocol serializer // based on `meta.path`. export type DumpStreamEvent = z.infer; - -// --- Storage shape (in-process, never serialized) --- - -export interface StoredDumpRequest { - method: string; - path: string; - // Captured verbatim with no redaction: the dump only surfaces to the - // owning API key's operator, who already holds the key. - headers: Array<[string, string]>; - body: Uint8Array; -} - -export type PreparedDumpRequestBody = { - readonly encoding: 'identity' | 'gzip'; - readonly bytes: Uint8Array; - readonly decodedByteLength: number; -}; - -export interface DumpWriteRequest { - method: string; - path: string; - headers: Array<[string, string]>; - body: PreparedDumpRequestBody; -} - -export type StoredDumpResponseBody = - | { type: 'stream'; events: DumpStreamEvent[] } - | { type: 'bytes'; body: Uint8Array } - | { type: 'none' }; - -export interface StoredDumpResponse { - status: number | null; - headers: Array<[string, string]>; - body: StoredDumpResponseBody; -} - -export type StoredDumpEdgeRecord = { - shape: 'edge'; - meta: DumpMetadata; - request: StoredDumpRequest; - response: StoredDumpResponse; -}; - // The run's NDJSON, still as bytes: it is a body file under the same contract // as the other two, gzipped into the file store and pointed at by the row's // descriptor. One `put` carries it whole — measured on production, P99 of a // turn's request and response together is 2.86 MB, well under the 5 MiB below // which multipart has nothing to divide. -export type StoredDumpRunRecord = { - shape: 'run'; +export type StoredDumpRecord = { meta: DumpMetadata; events: Uint8Array; }; -export type StoredDumpRecord = StoredDumpEdgeRecord | StoredDumpRunRecord; - -export type DumpWriteEdgeRecord = { - shape: 'edge'; - meta: DumpMetadata; - request: DumpWriteRequest; - response: StoredDumpResponse; -}; - -// The run half is the stored shape unchanged: a run's stream is encoded once the -// run is over, so there is nothing to compress ahead of the terminal write the -// way a request body is. -export type DumpWriteRecord = DumpWriteEdgeRecord | StoredDumpRunRecord; - -// --- Wire shape (serialized JSON over the dashboard's control plane) --- - -// `utf8` is chosen from the upstream content-type, with a UTF-8-fatal -// fallback to `base64` when a textual content-type carried non-UTF-8 bytes. -// JSON cannot carry `Uint8Array` directly, so this discriminator lives at -// the HTTP boundary and nowhere else. -export type DumpBody = - | { encoding: 'utf8'; data: string } - | { encoding: 'base64'; data: string }; - -interface DumpRequest { - method: string; - path: string; - headers: Array<[string, string]>; - body: DumpBody; -} - -export type DumpResponseBody = - | { type: 'stream'; events: DumpStreamEvent[] } - | { type: 'bytes'; body: DumpBody } - | { type: 'none' }; - -interface DumpResponse { - status: number | null; - headers: Array<[string, string]>; - body: DumpResponseBody; -} - -export type DumpEdgeRecord = { - shape: 'edge'; - meta: DumpMetadata; - request: DumpRequest; - response: DumpResponse; -}; - +// A run's stream is encoded once the run is over, so there is nothing to +// compress ahead of the terminal write the way a request body was. +export type DumpWriteRecord = StoredDumpRecord; // The stored NDJSON verbatim, decoded as UTF-8. One line is one event and one // SSE `data:` payload, so what a reader parses here is what a live observer // will parse frame by frame once the fan-out exists. -export type DumpRunRecord = { - shape: 'run'; +export type DumpRecord = { meta: DumpMetadata; events: string; }; - -export type DumpRecord = DumpEdgeRecord | DumpRunRecord; diff --git a/packages/gateway/src/dump/wire.ts b/packages/gateway/src/dump/wire.ts index a7d3b84518..7184509d2d 100644 --- a/packages/gateway/src/dump/wire.ts +++ b/packages/gateway/src/dump/wire.ts @@ -1,60 +1,10 @@ -import type { - DumpBody, - DumpRecord, - DumpResponseBody, - StoredDumpRecord, - StoredDumpResponseBody, -} from './types.ts'; -import { encodeBase64, isTextualMediaType } from '@floway-dev/protocols/common'; - -const contentTypeOf = (headers: ReadonlyArray): string => - headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''; - -// Wire encoding decision: textual content-types try UTF-8 first and fall -// back to base64 when the bytes do not decode cleanly (a content-type -// that lied about being text). -const encodeBodyForWire = (bytes: Uint8Array, contentType: string): DumpBody => { - if (isTextualMediaType(contentType)) { - try { - return { encoding: 'utf8', data: new TextDecoder('utf-8', { fatal: true }).decode(bytes) }; - } catch {} - } - return { encoding: 'base64', data: encodeBase64(bytes) }; -}; - -const responseBodyToWire = (body: StoredDumpResponseBody, contentType: string): DumpResponseBody => { - switch (body.type) { - case 'stream': return { type: 'stream', events: body.events }; - case 'bytes': return { type: 'bytes', body: encodeBodyForWire(body.body, contentType) }; - case 'none': return { type: 'none' }; - } -}; - -// Sole place the storage shape crosses into the wire shape. Called once, -// at the control-plane HTTP boundary, just before `c.json(...)`. A run's stream -// is decoded UTF-8-fatal rather than sniffed: the gateway wrote those bytes -// itself, so anything that fails to decode is a corrupted record and says so. -export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => { - if (record.shape === 'run') { - return { - shape: 'run', - meta: record.meta, - events: new TextDecoder('utf-8', { fatal: true }).decode(record.events), - }; - } - return { - shape: 'edge', - meta: record.meta, - request: { - method: record.request.method, - path: record.request.path, - headers: record.request.headers, - body: encodeBodyForWire(record.request.body, contentTypeOf(record.request.headers)), - }, - response: { - status: record.response.status, - headers: record.response.headers, - body: responseBodyToWire(record.response.body, contentTypeOf(record.response.headers)), - }, - }; -}; +import type { DumpRecord, StoredDumpRecord } from './types.ts'; + +// Sole place the storage shape crosses into the wire shape. Called once, at the control-plane +// HTTP boundary, just before `c.json(...)`. A run's stream is decoded UTF-8-fatal rather than +// sniffed: the gateway wrote those bytes itself, so anything that fails to decode is a corrupted +// record and says so. +export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => ({ + meta: record.meta, + events: new TextDecoder('utf-8', { fatal: true }).decode(record.events), +}); diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index 4437589425..7356a11aef 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -2,12 +2,8 @@ import { DUMP_FILE_PREFIX, SPILLED_FILE_STAGE_GRACE_MS } from './spilled-files-p import { parseUpstreamHue, parseUpstreamKind } from './upstream-parse.ts'; import { decodeDumpBodyDescriptor, - decodeDumpHeaders, - decodeDumpStreamEvents, decodePersistedDumpMetadata, encodeDumpBodyDescriptor, - encodeDumpHeaders, - encodeDumpStreamEvents, encodePersistedDumpMetadata, } from '../dump/storage-codec.ts'; import type { DumpBodyDescriptor } from '../dump/storage-codec.ts'; @@ -17,11 +13,7 @@ import type { DumpRecordId, DumpUpstreamRef, DumpWriteRecord, - PreparedDumpRequestBody, StoredDumpRecord, - StoredDumpRequest, - StoredDumpResponse, - StoredDumpResponseBody, } from '../dump/types.ts'; import { gunzipBytes, gzipBytes } from '../shared/gzip.ts'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; @@ -86,16 +78,6 @@ const putRawBody = async ( return { key, type }; }; -const putPreparedBody = async ( - files: FileStore, - key: string, - prepared: PreparedDumpRequestBody, -): Promise => { - const gz = prepared.encoding === 'gzip' ? prepared.bytes : await gzipBytes(prepared.bytes); - await files.put(key, gz); - return { key, type: 'bytes' }; -}; - const fetchBody = async (files: FileStore, descriptor: DumpBodyDescriptor): Promise => { const gz = await files.get(descriptor.key); if (!gz) throw new Error(`dump body missing for key=${descriptor.key}`); @@ -111,34 +93,13 @@ const NO_EDGE_HEADERS = '[]'; export class FileDumpStore implements DumpStore { constructor(private readonly db: SqlDatabase, private readonly files: FileStore) {} - async prepareRequestBody(body: Uint8Array): Promise { - return { - encoding: 'gzip', - bytes: await gzipBytes(body), - decodedByteLength: body.byteLength, - }; - } - - // Both shapes take the same route: files are staged in the registry, written, - // and only then pointed at by a row. A run's NDJSON is one more body file - // under that contract, carried by the response descriptor — which is what - // leaves retention, the sweep and the files-before-row ordering untouched by - // its arrival. + // A run's NDJSON is one body file, staged in the registry, written, and only + // then pointed at by a row — the contract that leaves retention, the sweep and + // the files-before-row ordering saying what they always said. async put(keyId: string, record: DumpWriteRecord): Promise { const bucket = hourBucket(record.meta.completedAt); - const requestFileKey = record.shape === 'run' || record.request.body.decodedByteLength === 0 - ? null - : bodyPath(keyId, bucket, record.meta.id, 'req'); - const responseFileKey = record.shape === 'run' - ? bodyPath(keyId, bucket, record.meta.id, 'run') - : record.response.body.type === 'none' - || (record.response.body.type === 'bytes' && record.response.body.body.byteLength === 0) - ? null - : bodyPath(keyId, bucket, record.meta.id, 'resp'); - const staged = [ - ...(requestFileKey === null ? [] : [{ fileKey: requestFileKey, ownerKind: 'dump-request' }]), - ...(responseFileKey === null ? [] : [{ fileKey: responseFileKey, ownerKind: 'dump-response' }]), - ]; + const responseFileKey = bodyPath(keyId, bucket, record.meta.id, 'run'); + const staged = [{ fileKey: responseFileKey, ownerKind: 'dump-response' }]; if (staged.length > 0) { await this.db .prepare( @@ -155,27 +116,7 @@ export class FileDumpStore implements DumpStore { .run(); } - let requestDescriptor: DumpBodyDescriptor | null = null; - let responseDescriptor: DumpBodyDescriptor | null = null; - if (record.shape === 'run') { - responseDescriptor = await putRawBody(this.files, responseFileKey!, record.events, 'run'); - } else { - if (requestFileKey !== null) { - requestDescriptor = await putPreparedBody(this.files, requestFileKey, record.request.body); - } - if (record.response.body.type === 'bytes') { - if (record.response.body.body.byteLength > 0) { - responseDescriptor = await putRawBody(this.files, responseFileKey!, record.response.body.body, 'bytes'); - } - } else if (record.response.body.type === 'stream') { - responseDescriptor = await putRawBody( - this.files, - responseFileKey!, - new TextEncoder().encode(encodeDumpStreamEvents(record.response.body.events, `dump record ${record.meta.id} response events`)), - 'events', - ); - } - } + const responseDescriptor = await putRawBody(this.files, responseFileKey, record.events, 'run'); // Files before row — a partial failure leaves orphan files the sweep // collects, never an orphan row whose detail fetch would 404. @@ -189,15 +130,9 @@ export class FileDumpStore implements DumpStore { record.meta.completedAt, record.meta.upstream?.id ?? null, encodePersistedDumpMetadata(record.meta, `dump record ${record.meta.id} metadata`), - record.shape === 'run' - ? NO_EDGE_HEADERS - : encodeDumpHeaders(record.request.headers, `dump record ${record.meta.id} request headers`), - record.shape === 'run' || record.response.body.type === 'none' - ? null - : encodeDumpHeaders(record.response.headers, `dump record ${record.meta.id} response headers`), - requestDescriptor === null - ? null - : encodeDumpBodyDescriptor(requestDescriptor, `dump record ${record.meta.id} request body descriptor`), + NO_EDGE_HEADERS, + null, + null, responseDescriptor === null ? null : encodeDumpBodyDescriptor(responseDescriptor, `dump record ${record.meta.id} response body descriptor`), @@ -250,55 +185,16 @@ export class FileDumpStore implements DumpStore { ...decodePersistedDumpMetadata(row.meta_json, `dump record ${recordId} metadata`), upstream: hydrateUpstream(row), }; - const requestDescriptor = row.request_body_descriptor === null - ? null - : decodeDumpBodyDescriptor(row.request_body_descriptor, `dump record ${recordId} request body descriptor`); const responseDescriptor = row.response_body_descriptor === null ? null : decodeDumpBodyDescriptor(row.response_body_descriptor, `dump record ${recordId} response body descriptor`); - - // The body kind is the shape: a run was written as one NDJSON stream and - // has no edge halves to rebuild. - if (responseDescriptor?.type === 'run') { - return { shape: 'run', meta, events: await fetchBody(this.files, responseDescriptor) }; - } - - const requestHeaders = decodeDumpHeaders(row.request_headers_json, `dump record ${recordId} request headers`); - const responseHeaders = row.response_headers_json === null - ? null - : decodeDumpHeaders(row.response_headers_json, `dump record ${recordId} response headers`); - - const request: StoredDumpRequest = { - method: meta.method, - path: meta.path, - headers: requestHeaders, - body: requestDescriptor ? await fetchBody(this.files, requestDescriptor) : new Uint8Array(), - }; - - // Headers null iff `type: 'none'`; a null descriptor with headers is a - // legitimate empty-body `bytes` response (nothing to gzip), reconstructed - // here from a zero-length buffer so the discriminator round-trips. - let responseBody: StoredDumpResponseBody; - if (responseHeaders === null) { - responseBody = { type: 'none' }; - } else if (responseDescriptor === null) { - responseBody = { type: 'bytes', body: new Uint8Array() }; - } else if (responseDescriptor.type === 'events') { - const text = new TextDecoder().decode(await fetchBody(this.files, responseDescriptor)); - responseBody = { - type: 'stream', - events: decodeDumpStreamEvents(text, `dump record ${recordId} response events at key=${responseDescriptor.key}`), - }; - } else { - responseBody = { type: 'bytes', body: await fetchBody(this.files, responseDescriptor) }; + // A record is one NDJSON stream, carried by the response descriptor and saying so. Anything + // else is a row from before the shape that produced it was deleted, which the migration that + // deleted them leaves none of — so it is corruption rather than a second shape. + if (responseDescriptor?.type !== 'run') { + throw new Error(`dump record ${recordId} has no run stream to read`); } - - const response: StoredDumpResponse = { - status: meta.status, - headers: responseHeaders ?? [], - body: responseBody, - }; - return { shape: 'edge', meta, request, response }; + return { meta, events: await fetchBody(this.files, responseDescriptor) }; } async deleteExpiredBatch(keyId: string, now: number, limit: number): Promise { diff --git a/packages/pipeline/src/dump.ts b/packages/pipeline/src/dump.ts index 5811a7dace..3af678134f 100644 --- a/packages/pipeline/src/dump.ts +++ b/packages/pipeline/src/dump.ts @@ -299,3 +299,60 @@ export const encodeRun = (events: readonly Event[], options?: { readonly shareSt * so the stored file and the live protocol are the same bytes. */ export const toNdjson = (events: readonly DumpEvent[]): string => events.map(event => `${JSON.stringify(event)}\n`).join(''); + +// --- The reading ------------------------------------------------------------------- + +/** + * Resolves a run's own object space as its events arrive. + * + * The encoder interns: a value that appears twice is written once as a node under an `object` + * event and pointed at by `{"$": n}` everywhere after. A reader folds the same way — it keeps + * every node it has been given and resolves a reference against them — and it can do so line by + * line, because the format's own invariant is that a reference never points forward. + * + * What comes back is the value as the run held it, with the two things JSON cannot carry + * restored: `undefined` and the numeric edges. A secret stays redacted, because redaction is + * what was stored; bytes stay base64, because reading them back out is a decision about what + * they are and this knows only that they are bytes. + */ +export const createRunReader = () => { + const nodes = new Map(); + const resolved = new Map(); + + const read = (stored: Stored): unknown => { + if (stored === null || typeof stored !== 'object') return stored; + if (Array.isArray(stored)) return stored.map(read); + if ('$' in stored) { + const id = stored.$ as number; + if (resolved.has(id)) return resolved.get(id); + const node = nodes.get(id); + if (node === undefined) throw new Error(`dump reference $${id} points at a node no event carried`); + const value = read(node); + resolved.set(id, value); + return value; + } + if ('$stream' in stored) return { stream: stored.$stream }; + if ('$secret' in stored) return stored.$secret; + if ('$bytes' in stored) return { bytes: stored.$bytes }; + if ('$undefined' in stored) return undefined; + if ('$number' in stored) return stored.$number === 'NaN' ? NaN : stored.$number === 'Infinity' ? Infinity : -Infinity; + if ('$bigint' in stored) return BigInt(stored.$bigint as string); + return Object.fromEntries(Object.entries(stored).map(([key, child]) => [decodeKey(key), read(child)])); + }; + + /** What one event says. An `object` event says nothing on its own — it is the space the + * events around it are written against — so it reads as `null`. */ + return (event: DumpEvent): { readonly facts?: Record; readonly frames?: readonly unknown[] } | null => { + if (event.type === 'object') { + event.nodes.forEach((node, index) => { nodes.set(event.fromObjectId + index, node); }); + return null; + } + if (event.type === 'stream.frame') return { frames: event.frames.map(read) }; + if (event.type === 'stage.entered' || event.type === 'stage.leaved') { + return event.facts === undefined + ? {} + : { facts: Object.fromEntries(Object.entries(event.facts).map(([key, value]) => [decodeKey(key), read(value)])) }; + } + return {}; + }; +}; diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index d6604b1fff..aff36222c6 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -34,6 +34,7 @@ export type { Deferred, Owned, RunResult } from './run.ts'; export { createEncoder, createRunEncoder, + createRunReader, decodeKey, encodeKey, encodeRun, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51094c6e2a..c074067c06 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -139,6 +139,9 @@ importers: apps/web: dependencies: + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../../packages/pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../../packages/protocols From 6501d09b79e58b3dab7e242d88e4b9ee4d4d6914 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 11:51:17 +0800 Subject: [PATCH 093/156] feat(openai-responses): make a shim's image call a run of its own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruling 2-and-6's second half: a server tool's backend call is not part of the turn that asked for it — it is an independent run, with its own prologue, its own settlement and its own record. The dial is unchanged. It still resolves the pinned image model, still carries the rate-limit retry a hosted tool needs, still produces the lifecycle events the caller splices into its answer. What changes is where its cost lands: it used to call `recordTokenUsage` and `recordPerformance` beside the dial, and it now hands up a reading that `writeSettlement` writes both rows from — the same seam every other upstream call settles through. Its record has to be its own, and that is forced rather than chosen: a second run numbers its stages from 1 again, so events landing in the parent's record would collide with the parent's ids and read as one turn entering a stage twice. `RunDump.openSubRequest` opens the sibling — same key, same scheduler, so it is retained and swept by the same rule. Its attempt slot is its own too, which is what the hand-rolled version was already careful about: the performance sample is attributed to the sub-run's slot, so the turn that asked for the image keeps its own upstream stamp. A sub-request has no seam to settle it, so it settles itself after the lifecycle drains — the reading resolves when the events run out, which is after the run answered. `Deferred` earned its keep here. A resolution failure returned the terminal without ever reaching the settlement path, and the reading it had promised never settled; three tests hit the teardown deadline and named it. That path settles now, with nothing billed and no performance context, because a call that resolved no candidate has still finished. With one recording shape left, `TurnDump` was the last remnant of the era that had two — an interface between an accumulator that no longer exists and the run recording. It is folded into `RunDump`, and `recordStream` with it. --- .../image-generation-integration_test.ts | 15 +- .../chat/anthropic-messages/pipeline.ts | 2 +- .../chat/gemini-generate-content/pipeline.ts | 2 +- .../chat/openai-chat-completions/pipeline.ts | 2 +- .../chat/openai-responses/compact.ts | 2 +- .../chat/openai-responses/pipeline.ts | 2 +- .../server-tools/image-generation.ts | 76 ++++++---- .../server-tools/image-sub-request.ts | 135 ++++++++++++++++++ .../chat/openai-responses/websocket.ts | 8 +- .../src/data-plane/openai-audio/pipeline.ts | 2 +- .../data-plane/openai-completions/pipeline.ts | 4 +- .../src/data-plane/openai-images/pipeline.ts | 2 +- .../src/data-plane/shared/gateway-ctx.ts | 6 +- packages/gateway/src/dump/run-sink.ts | 93 +++++++++++- packages/gateway/src/dump/turn-dump.ts | 106 -------------- 15 files changed, 304 insertions(+), 153 deletions(-) create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts delete mode 100644 packages/gateway/src/dump/turn-dump.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts index cc25d93061..51f97fd4e4 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-generation-integration_test.ts @@ -97,8 +97,8 @@ const emptyResult = (status: OpenAIResponsesResult['status']): OpenAIResponsesRe id: 'upstream', object: 'response', model: 'orchestrator', output: [], output_text: '', status, error: null, incomplete_details: null, }); -const jsonResponse = (b64: string): Response => - new Response(JSON.stringify({ data: [{ b64_json: b64 }] }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); +const jsonResponse = (b64: string, usage?: Record): Response => + new Response(JSON.stringify({ data: [{ b64_json: b64 }], ...(usage === undefined ? {} : { usage }) }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); const REMOTE_PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; const remotePngResponse = (): Response => new Response(Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); @@ -574,7 +574,7 @@ test('resolveImageCandidate renders model_not_supported when image-kind candidat }); test('an image sub-call records its own perf row attributed to the image backend, leaving the outer attempt untouched', async () => { - stub.nextGenerations = [jsonResponse('R0VO')]; + stub.nextGenerations = [jsonResponse('R0VO', { input_tokens: 9, output_tokens: 4, total_tokens: 13 })]; // Real scheduler: capture the promises the shim fires so the test can // await them before querying the repo (the default no-op scheduler in // `gatewayCtx()` would drop the recordSample write). @@ -590,12 +590,21 @@ test('an image sub-call records its own perf row attributed to the image backend await drain(result); await Promise.all(pending); + // Through settlement, like every other upstream call: the row is written from the reading the + // sub-run hands up rather than by a telemetry call beside the dial. const perfRows = await repo.performance.listAll(); const imageRows = perfRows.filter(r => r.operation === 'image_generation'); assertEquals(imageRows.length, 1); assertEquals(imageRows[0].upstream, 'u'); assertEquals(imageRows[0].keyId, 'key_test'); assertEquals(imageRows[0].model, 'gpt-image-2'); + + // And the usage row beside it, from the same reading — one settlement writes both, which is + // what a sub-request being a run rather than a dial with telemetry beside it buys. + const usageRows = await repo.usage.listAll(); + const imageUsage = usageRows.filter(row => row.model === 'gpt-image-2'); + assertEquals(imageUsage.length, 1); + assertEquals(imageUsage[0]!.upstream, 'u'); // The image shim runs on a local AttemptState distinct from the outer // OpenAI Responses turn's — no image-call stamps may leak onto ctx.attempt. assertEquals(ctx.attempt.upstreamCallStartedAt, null); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 05e018f6d8..c35a608393 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -25,7 +25,7 @@ import { isClaudeCodeProbe, probeFrames } from './claude-code-probe.ts'; import { renderAnthropicMessagesError } from './errors.ts'; import { createAnthropicMessagesBillableUsageReader } from './usage.ts'; import { runAnthropicMessagesWebSearchTool } from './web-search-tool.ts'; -import { recordStream, streamReferenceOf } from '../../../dump/turn-dump.ts'; +import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index 0a21fb8326..47d0b8ca7d 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -48,7 +48,7 @@ import { wrapGeminiGenerateContentAffinityEgress } from './affinity/egress.ts'; import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; import { renderGeminiGenerateContentError } from './errors.ts'; -import { recordStream, streamReferenceOf } from '../../../dump/turn-dump.ts'; +import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index 09729d0f4f..1832304039 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -23,7 +23,7 @@ import { wrapOpenAIChatCompletionsAffinityEgress } from './affinity/egress.ts'; import { analyzeOpenAIChatCompletionsAffinity } from './affinity/ingress.ts'; import { billableUsageFromOpenAIChatCompletionsEvent } from './usage.ts'; -import { recordStream, streamReferenceOf } from '../../../dump/turn-dump.ts'; +import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index ceb74e2fbd..e9416220d2 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -61,7 +61,7 @@ import { imageGenerationServerTool } from './server-tools/image-generation.ts'; import { runOpenAIResponsesServerTools } from './server-tools/stage.ts'; import { webSearchServerTool } from './server-tools/web-search.ts'; import { billableUsageFromOpenAIResponsesResult } from './usage.ts'; -import { recordStream } from '../../../dump/turn-dump.ts'; +import { recordStream } from '../../../dump/run-sink.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 8c205f536f..5d36198f96 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -57,7 +57,7 @@ import { imageGenerationServerTool } from './server-tools/image-generation.ts'; import { runOpenAIResponsesServerTools } from './server-tools/stage.ts'; import { webSearchServerTool } from './server-tools/web-search.ts'; import { billableUsageFromOpenAIResponsesEvent, billableUsageFromOpenAIResponsesResult } from './usage.ts'; -import { recordStream, streamReferenceOf } from '../../../dump/turn-dump.ts'; +import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { AttemptSelector, BillableEntity } from '../../pipeline/facts.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts index c88b4a2e7e..204e6515cc 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-generation.ts @@ -1,10 +1,12 @@ +import { runImageGenerationSubRequest, type SettleImageCall } from './image-sub-request.ts'; import type { ServerToolLifecycleEvent, ServerToolOutputItem, ServerToolRegistration, ServerToolTerminal } from './shim.ts'; import { sleep } from '../../../../shared/sleep.ts'; +import type { BillableEntity } from '../../../pipeline/facts.ts'; import { enumerateModelCandidates } from '../../../providers/resolution.ts'; import { appendFailedUpstreams } from '../../../shared/failed-upstreams.ts'; -import { stampUpstreamCallStart, type AttemptState } from '../../../shared/gateway-ctx.ts'; -import { recordPerformance, type PerformanceTelemetryContext } from '../../../shared/telemetry/performance.ts'; -import { recordTokenUsage, tokenUsageFromOpenAIImagesBody } from '../../../shared/telemetry/usage.ts'; +import { stampUpstreamCallStart, type AttemptState, type GatewayCtx } from '../../../shared/gateway-ctx.ts'; +import type { PerformanceTelemetryContext } from '../../../shared/telemetry/performance.ts'; +import { tokenUsageFromOpenAIImagesBody, tokenUsageMeasurement } from '../../../shared/telemetry/usage.ts'; import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../shared/external-image-loader.ts'; import { dimensionsFromBytes, getImageProcessor, type BackgroundScheduler } from '@floway-dev/platform'; import { decodeForgivingBase64, encodeHex, isImageMediaType, mediaTypeEssence, parseSSEStream } from '@floway-dev/protocols/common'; @@ -901,20 +903,21 @@ interface ShimState { runtimeLocation: string; downstreamAbortSignal: AbortSignal | undefined; imageDispatchCount: number; + /** The turn's own context, which each call opens its own run over. */ + gateway: GatewayCtx; } -const recordImageUsage = (state: ShimState, provider: Provider, model: ProviderModel, modelKey: string, responseBody: unknown): void => { +/** What one backend answer is billable for, in the shape settlement writes a row from. A body + * that reported nothing bills nothing, which is a different statement from reporting zero. */ +const billedImage = (provider: Provider, model: ProviderModel, modelKey: string, responseBody: unknown): readonly BillableEntity[] => { const usage = tokenUsageFromOpenAIImagesBody(responseBody); - if (usage === null) return; - const promise = recordTokenUsage(state.apiKeyId, { - model: model.id, - upstream: provider.upstreamId, - modelKey, - pricing: model.pricing ?? null, - }, usage).catch((error: unknown) => { - console.error('Failed to record image generation usage:', error); - }); - state.backgroundScheduler(promise); + if (usage === null) return []; + const measurement = tokenUsageMeasurement(usage); + return [{ + identity: { model: model.id, upstream: provider.upstreamId, modelKey, pricing: model.pricing ?? null }, + quantities: measurement.quantities, + pricingFacts: measurement.pricingFacts, + }]; }; const buildGenerationsBody = (prompt: string, config: ImageGenerationConfig, stream: boolean): Record => ({ @@ -1115,6 +1118,7 @@ const consumeImageResponse = async ( modelKey: string, response: Response, state: ShimState, + billed: BillableEntity[], ): Promise => { const text = await response.text(); if (!response.ok) { @@ -1137,7 +1141,7 @@ const consumeImageResponse = async ( if (b64 === null) { return { ok: false, error: { type: 'image_generation_error', message: 'Image backend response did not contain image bytes.', code: 'server_error', retryable: true } }; } - recordImageUsage(state, provider, model, modelKey, parsed); + billed.push(...billedImage(provider, model, modelKey, parsed)); return { ok: true, b64, echo: extractEcho(parsed) }; }; @@ -1236,9 +1240,17 @@ const streamImageGeneration = ( isEdit: boolean, sources: readonly ImageSource[], state: ShimState, + settle: SettleImageCall, + billed: BillableEntity[], ) => async function* (): AsyncGenerator { const resolved = await resolveImageCandidate(isEdit, state); - if (!resolved.ok) return imageTerminal(prompt, action, { ok: false, error: resolved.error }); + if (!resolved.ok) { + // No candidate means no upstream was called, so there is nothing billed and no performance + // context to attribute — but the reading still settles, because a run waits for what it + // started and a call that resolved nothing has finished. + settle([], true, undefined); + return imageTerminal(prompt, action, { ok: false, error: resolved.error }); + } const { provider, fetcher } = resolved.candidate; const model = providerModelOf(resolved.candidate); const wantsPartials = (state.config.partial_images ?? 0) > 0; @@ -1251,8 +1263,11 @@ const streamImageGeneration = ( operation: isEdit ? 'image_edit' : 'image_generation', runtimeLocation: state.runtimeLocation, }; + // The call's own settlement, through the seam every other upstream call settles by: the + // performance sample is attributed to this run's attempt slot rather than the turn's, which is + // what keeps the outer turn's upstream stamp intact. const finish = (outcome: ImageOutcome): ServerToolTerminal => { - recordPerformance({ attempt, backgroundScheduler: state.backgroundScheduler }, perfContext, !outcome.ok, 0, performance.now()); + settle(billed, !outcome.ok, perfContext); return imageTerminal(prompt, action, outcome); }; @@ -1280,7 +1295,7 @@ const streamImageGeneration = ( } if (!wantsPartials) { - return finish(await consumeImageResponse(provider, model, modelKey, response, state)); + return finish(await consumeImageResponse(provider, model, modelKey, response, state, billed)); } if (!response.ok) { @@ -1310,7 +1325,7 @@ const streamImageGeneration = ( if (finalB64 === undefined) { return finish({ ok: false, error: { type: 'image_generation_error', message: 'Image backend stream ended without a completed image.', code: 'server_error', retryable: true } }); } - recordImageUsage(state, provider, model, modelKey, { usage }); + billed.push(...billedImage(provider, model, modelKey, { usage })); return finish({ ok: true, b64: finalB64, echo: finalEcho }); }; @@ -1474,6 +1489,7 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati const state: ShimState = { config: materializedConfig, + gateway: gatewayCtx, apiKeyId: gatewayCtx.apiKeyId, upstreamIds: gatewayCtx.upstreamIds, backgroundScheduler: gatewayCtx.backgroundScheduler, @@ -1536,13 +1552,21 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati { type: 'response.image_generation_call.in_progress' }, { type: 'response.image_generation_call.generating' }, ], - run: streamImageGeneration( - promptArg, - operation.action, - operation.action === 'edit', - sources, - state, - ), + // Its own run: its own prologue, its own settlement, its own record. What this + // yields is that run's lifecycle, spliced into the turn that asked for it. + run: () => (async function* (): AsyncGenerator { + const billed: BillableEntity[] = []; + const call = await runImageGenerationSubRequest( + state.gateway, + operation.action, + (settle: SettleImageCall) => streamImageGeneration(promptArg, operation.action, operation.action === 'edit', sources, state, settle, billed)(), + ); + try { + return yield* call.lifecycle; + } finally { + await call.drain(); + } + })(), }]; }, }, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts new file mode 100644 index 0000000000..6e378020f3 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts @@ -0,0 +1,135 @@ +// The image the shim generates, as a run of its own. +// +// Ruling 2-and-6: a server tool's backend call is not part of the turn that asked for it — it is +// an independent run, with its own prologue, its own settlement and its own record. Its stages +// number from 1 like any run's, which is why the record has to be its own: events landing in the +// parent's would collide with the parent's ids and read as one turn entering a stage twice. +// +// What the ending does is unchanged — resolve the pinned image model, dial it with the rate-limit +// retry the hosted tool needs, and produce the lifecycle events the caller splices. What changes +// is where its cost lands: the reading it settles is the one `writeSettlement` writes both rows +// from, so a shim call bills through the same seam every other upstream call does rather than +// through two hand-rolled telemetry calls beside it. + +import type { ServerToolLifecycleEvent, ServerToolTerminal } from './shim.ts'; +import { consoleLogSink } from '../../../../runtime/log.ts'; +import type { BillableEntity, GatewayFacts } from '../../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../../pipeline/serve.ts'; +import { prologueFor } from '../../../pipeline/serve.ts'; +import type { GatewayServices } from '../../../pipeline/services.ts'; +import { settleBillable, writeSettlement } from '../../../pipeline/settlement.ts'; +import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; +import type { PerformanceTelemetryContext } from '../../../shared/telemetry/performance.ts'; +import { compose, defer, defineStage, move, run, type Deferred, type Pipeline } from '@floway-dev/pipeline'; + +/** What one image call is, and what it comes to. */ +export interface ImageSubRequestFacts extends GatewayFacts { + /** The call, already planned: everything the ending needs to dial and nothing about the turn + * that asked for it. */ + 'request.imageGeneration.call': (settle: SettleImageCall) => AsyncGenerator; + /** The lifecycle the caller splices into its own answer. */ + 'response.imageGeneration.lifecycle': AsyncGenerator; + /** What the call turned out to cost, once its events have run out — which is after this run + * has answered, because the caller is what drives them. */ + 'response.imageGeneration.streamedUsage': Deferred | null; +} + +type I = { [P in K]: ImageSubRequestFacts[P] }; + +/** How the ending reports what it did, once it knows. The identity and the counts arrive with + * the backend's own response, long after this run handed up. */ +export type SettleImageCall = (billable: readonly BillableEntity[], failed: boolean, telemetry: PerformanceTelemetryContext | undefined) => void; + +/** + * The ending. It hands up the lifecycle the caller drives, and the reading that lifecycle + * settles — the same shape every streaming family hands up, for the same reason: the numbers + * arrive with the last event. + */ +const dialImageGeneration = defineStage< + I<'request.imageGeneration.call'>, + I<'response.imageGeneration.lifecycle' | 'response.imageGeneration.streamedUsage'> & { 'response.usage.billable': readonly BillableEntity[] }, + GatewayServices +>({ + name: 'dialImageGeneration', + return: { + provides: ['response.imageGeneration.lifecycle', 'response.imageGeneration.streamedUsage', 'response.usage.billable'], + }, + execute: async (facts, use) => { + let settle!: (outcome: StreamOutcome) => void; + // Declared as this run's own unfinished work, so the runner waits for it at teardown where + // it can see it rather than the reading being started and forgotten. + const outcome = defer(new Promise(resolve => { settle = resolve; })); + const call = facts['request.imageGeneration.call']; + return move({ + ...facts, + 'response.imageGeneration.lifecycle': call((billable, failed, telemetry) => { + // The sample is attributed to this run's own attempt slot, which is what keeps the + // turn that asked for the image from having its upstream stamp overwritten. + use.gateway.attempt.telemetry = telemetry; + settle({ billable, failed }); + }), + 'response.imageGeneration.streamedUsage': outcome, + // Nothing has been reported when this hands up; what the call turns out to have cost + // arrives through the reading above. + 'response.usage.billable': [], + }) as never; + }, +}); + +export const imageGenerationSubRequestPipeline: Pipeline< + I<'request.imageGeneration.call'>, + I<'response.imageGeneration.lifecycle' | 'response.imageGeneration.streamedUsage'> +> = compose('imageGenerationSubRequest', [ + writeSettlement( + () => false, + handedUp => (handedUp as { 'response.imageGeneration.streamedUsage'?: unknown })['response.imageGeneration.streamedUsage'] !== null, + ), + dialImageGeneration, +]); + +/** + * The run a shim call is. + * + * Its context is the parent's, with the three things a separate run owes itself: a start time of + * its own, an attempt slot of its own — so the outer turn's upstream stamp survives — and a + * record of its own. + */ +export const runImageGenerationSubRequest = async ( + parent: GatewayCtx, + action: 'generate' | 'edit', + call: ImageSubRequestFacts['request.imageGeneration.call'], +): Promise<{ + readonly lifecycle: AsyncGenerator; + readonly drain: () => Promise; +}> => { + const path = action === 'edit' ? '/images/edits' : '/images/generations'; + const dump = parent.dump?.openSubRequest({ method: 'POST', path }) ?? null; + const gateway: GatewayCtx = { + ...parent, + requestStartedAt: Date.now(), + attempt: { firstOutputTokenAt: null, upstreamCallStartedAt: null, telemetry: undefined }, + dump, + }; + const prologue = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }, dump); + + const { facts, drain } = await run( + imageGenerationSubRequestPipeline, + move({ 'request.imageGeneration.call': call }) as never, + prologue.services as never, + ); + return { + lifecycle: facts['response.imageGeneration.lifecycle'], + // The epilogue a served turn gets from the seam, which a sub-request has to be its own: the + // reading resolves when the lifecycle runs out, which is after this run answered, so the row + // is written here rather than in the chain. + drain: async () => { + const reading = facts['response.imageGeneration.streamedUsage']; + if (reading !== null) { + const outcome = await reading; + settleBillable({ ...prologue.services, log: consoleLogSink }, outcome.billable, outcome.failed); + } + await drain(); + dump?.finalize(200, 0); + }, + }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index 1f37cc2ee2..91116a3f2e 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -13,7 +13,7 @@ import type { Context } from 'hono'; import { createOpenAIResponsesWsSession, type StatefulOpenAIResponsesStore } from './items/store.ts'; import { openaiResponsesServePipeline, type OpenAIResponsesFacts, type OpenAIResponsesServeExit } from './pipeline.ts'; import { openRunDump } from '../../../dump/run-sink.ts'; -import type { TurnDump } from '../../../dump/turn-dump.ts'; +import type { RunDump } from '../../../dump/run-sink.ts'; import { apiKeyFromContext, authenticateApiKey, type AuthedContext } from '../../../middleware/auth.ts'; import type { ApiKey } from '../../../repo/types.ts'; import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; @@ -767,7 +767,7 @@ const sendError = ( status: number, error: Record, eventId?: string, - dump?: TurnDump | null, + dump?: RunDump | null, ): void => { sendJson(socket, { type: 'error', status, error }, eventId, dump); }; @@ -784,14 +784,14 @@ const sendOpenAIResponsesEvent = ( socket: OpenAIResponsesWebSocketSocket, event: ClientOpenAIResponsesStreamEvent, eventId?: string, - dump?: TurnDump | null, + dump?: RunDump | null, ): boolean => sendJson(socket, event, eventId, dump); const sendJson = ( socket: OpenAIResponsesWebSocketSocket, value: unknown, eventId?: string, - dump?: TurnDump | null, + dump?: RunDump | null, ): boolean => { if (socket.readyState !== 1) return false; const payload = eventId === undefined || !value || typeof value !== 'object' diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index 88407b021c..8ee16aa8e9 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -13,7 +13,7 @@ // callOpenAIAudioTranscriptionUpstream the ending: dials, reads what came back, and // provides the answer plus what is billable -import { recordStream, streamReferenceOf } from '../../dump/turn-dump.ts'; +import { recordStream, streamReferenceOf } from '../../dump/run-sink.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index 1468139696..d854ed854e 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -13,7 +13,7 @@ // answer plus what is billable import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts'; -import { recordStream, streamReferenceOf, type TurnDump } from '../../dump/turn-dump.ts'; +import { recordStream, streamReferenceOf, type RunDump } from '../../dump/run-sink.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; @@ -253,7 +253,7 @@ const readUpstream = async ( identity: TelemetryModelIdentity, candidate: ModelCandidate, signal: AbortSignal | undefined, - dump: TurnDump | null, + dump: RunDump | null, ): Promise => { // An upstream was called and reported nothing — which is what every arm but a read usage // block leaves standing, and is a different statement from reporting zero. diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index 4151a93971..da16894e84 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -11,7 +11,7 @@ // failover runs what follows once per candidate // callOpenAIImagesUpstream the ending: dials, and provides what came back -import { recordStream, streamReferenceOf } from '../../dump/turn-dump.ts'; +import { recordStream, streamReferenceOf } from '../../dump/run-sink.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; diff --git a/packages/gateway/src/data-plane/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/shared/gateway-ctx.ts index 5d115b4894..9fbfeeddf0 100644 --- a/packages/gateway/src/data-plane/shared/gateway-ctx.ts +++ b/packages/gateway/src/data-plane/shared/gateway-ctx.ts @@ -1,5 +1,5 @@ import type { RequestBody } from './request-body.ts'; -import type { TurnDump } from '../../dump/turn-dump.ts'; +import type { RunDump } from '../../dump/run-sink.ts'; import { apiKeyFromContext, type AuthedContext, effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; @@ -40,7 +40,7 @@ export interface GatewayCtx { // Null when the api key has no dump retention configured, in which case // `finalizeGatewayResponse` short-circuits the dump tee and returns the // response untouched. - readonly dump: TurnDump | null; + readonly dump: RunDump | null; } export interface CreateGatewayCtxOptions { @@ -48,7 +48,7 @@ export interface CreateGatewayCtxOptions { // What this turn is recorded as, opened by whoever opened the run — every entry does, through // the prologue. `null` is a key with no retention configured, which is what makes recording // conditional: there is nothing to hand the runner rather than a sink that discards. - dump: TurnDump | null; + dump: RunDump | null; // WebSocket-style call sites own the AbortController (so the upgrade // handler can cancel mid-stream); HTTP call sites let the factory mint one // when wantsStream is true. diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 635432adc3..332167ea36 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -15,12 +15,11 @@ import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; -import type { StreamRecording } from './turn-dump.ts'; import type { DumpMetadata } from './types.ts'; import type { RequestBody } from '../data-plane/shared/request-body.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; -import { createRunEncoder, streamFact, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; +import { createRunEncoder, isStreamFact, streamFact, toNdjson, type DumpEvent, type Event, type StreamFact } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; @@ -104,6 +103,23 @@ export class RunDump { this.attribution.success(identity, usage); } + /** + * A record for a run this one started. + * + * A sub-request is an independent run — its own prologue, its own settlement — and its record + * has to be its own too: a second run numbers its stages from 1 again, so its events landing + * here would collide with this run's ids and the tree would read as one turn that entered the + * same stage twice. Same key, same scheduler, so it is retained and swept by the same rule. + */ + openSubRequest(turn: { readonly method: string; readonly path: string }): RunDump { + return new RunDump( + this.apiKey, + { method: turn.method, path: turn.path, bodyByteLength: 0, streamError: null }, + Date.now(), + this.backgroundScheduler, + ); + } + // --- terminal point --- // Two input shapes, matching the edge accumulator's seam: @@ -221,3 +237,76 @@ export const openRunDump = ( backgroundScheduler, ); }; + +/** + * One stream, as the recording knows it. + * + * A record identifies its streams, because their content arrives over time and after the fact + * that holds them: the fact carries `{"$stream": n}` and the frames arrive afterwards naming + * that id. `end` is what says the record of this stream is complete — a client that stopped + * reading leaves it short, and the absence of the terminator is how a reader tells a stream that + * ended from one that was cut off. + */ +export interface StreamRecording { + frame(frame: ProtocolFrame): void; + end(): void; + readonly fact: StreamFact; +} + +/** + * Records a stream's frames as they are read, and hands them on untouched. + * + * What the record holds is the stream as the client is served it, so the tee goes outside + * whatever shapes the frames and inside whatever frames them for a transport. For a family with + * translated wires that is its edge and nowhere lower: below the edge the frames are the + * upstream's and may still be another protocol's, above it they are transport frames the record + * does not describe. A non-streaming turn folds the same frames into one value, and recording + * here is what puts both in the record — the frames as they flowed beside the value assembled + * from them. + * + * Reading is what records: a losing attempt nobody read contributes nothing, because the + * release path drains the stream underneath this rather than through it, and a stream that + * stopped short is recorded as far as it got. + * + * The value handed back *is* the stream reference, so the fact that holds it encodes as + * `{"$stream": n}` and the frames that arrive afterwards say which stream they belong to. + * + * A record holds protocol frames, which is what most families' streams already carry. The + * family whose stream is bare protocol events says how one becomes a frame, because the record + * cannot guess and a cast would be it guessing. + */ +export function recordStream(stream: AsyncIterable>, dump: RunDump | null): AsyncIterable>; +export function recordStream(stream: AsyncIterable, dump: RunDump | null, asFrame: (value: T) => ProtocolFrame): AsyncIterable; +export function recordStream( + stream: AsyncIterable, + dump: RunDump | null, + asFrame: (value: T) => ProtocolFrame = value => value as ProtocolFrame, +): AsyncIterable { + // No recording configured hands the same iterable back, so a record shows no step where + // nothing happened and the stream is not wrapped for nobody. + if (dump === null) return stream; + + const recording = dump.openStream(); + return { + ...recording.fact, + [Symbol.asyncIterator]: () => (async function* () { + for await (const value of stream) { + recording.frame(asFrame(value)); + yield value; + } + // Reached only where the source ran out on its own, which is what makes the record of + // this stream complete. A reader that stopped early never gets here. + recording.end(); + })(), + }; +} + +/** + * The reference a value carries to the stream the record holds, for a wrapper to carry across. + * + * A stream is framed again on its way out — protocol frames become SSE — and what the client + * is handed is a different object over the same frames. Carrying the reference onto it is what + * lets the fact that produced the stream and the fact that framed it point at one record. + */ +export const streamReferenceOf = (value: unknown): StreamFact | Record => + isStreamFact(value) ? { ...value } : {}; diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts deleted file mode 100644 index 10c51c4c7a..0000000000 --- a/packages/gateway/src/dump/turn-dump.ts +++ /dev/null @@ -1,106 +0,0 @@ -// What a turn stamps on its recording, whichever shape that recording is. -// -// Two shapes are alive while the migration runs: the edge record, which holds what the client -// sent and what it got back, and the run record, which holds every stage of the pipeline as -// an event stream. The shape follows the endpoint — a pipelined one produces the run shape — -// and the stages in between neither know nor care which they are stamping. - -import type { TokenUsage } from '../repo/types.ts'; -import { isStreamFact, type StreamFact } from '@floway-dev/pipeline'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { TelemetryModelIdentity } from '@floway-dev/provider'; - -/** - * One stream, as the recording knows it. - * - * A run record identifies its streams, because their content arrives over time and after the - * fact that holds them: the fact carries `{"$stream": n}` and the frames arrive afterwards - * naming that id. `end` is what says the record of this stream is complete — a client that - * stopped reading leaves it short, and the absence of the terminator is how a reader tells a - * stream that ended from one that was cut off. - * - * The edge record has one frame log and no way to name anything in it, so its `fact` is null - * and its terminator has nothing to write. That is the shape's limit rather than an omission, - * and it goes away with the shape. - */ -export interface StreamRecording { - frame(frame: ProtocolFrame): void; - end(): void; - readonly fact: StreamFact | null; -} - -export interface TurnDump { - requestedModel(model: string): void; - success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void; - error(kind: 'upstream' | 'gateway', upstream?: string): void; - failed(reason: unknown): void; - frame(frame: ProtocolFrame): void; - /** Begins recording one stream. Every call is a new one, which is what lets a turn that - * opens two — a sub-request beside the answer — keep them apart. */ - openStream(): StreamRecording; - /** How much of the answer actually went out. A transport that writes its own frames counts - * them itself, because nothing downstream of it can. */ - recordSentPayloadBytes(byteLength: number): void; - /** Closes the recording. A transport that owns its own response — the WebSocket turn — - * states what it sent; an HTTP one hands over the response so the bytes can be teed. */ - finalize(status: number | null, responseBytes: number | readonly unknown[]): void; - finalize(response: Response): Response; -} - -/** - * Records a stream's frames as they are read, and hands them on untouched. - * - * What the record holds is the stream as the client is served it, so the tee goes outside - * whatever shapes the frames and inside whatever frames them for a transport. For a family with - * translated wires that is its edge and nowhere lower: below the edge the frames are the - * upstream's and may still be another protocol's, above it they are transport frames the record - * does not describe. A non-streaming turn folds the same frames into one value, and recording - * here is what puts both in the record — the frames as they flowed beside the value assembled - * from them. - * - * Reading is what records: a losing attempt nobody read contributes nothing, because the - * release path drains the stream underneath this rather than through it, and a stream that - * stopped short is recorded as far as it got. - * - * The value handed back *is* the stream reference, so the fact that holds it encodes as - * `{"$stream": n}` and the frames that arrive afterwards say which stream they belong to. - * - * A record holds protocol frames, which is what most families' streams already carry. The - * family whose stream is bare protocol events says how one becomes a frame, because the record - * cannot guess and a cast would be it guessing. - */ -export function recordStream(stream: AsyncIterable>, dump: TurnDump | null): AsyncIterable>; -export function recordStream(stream: AsyncIterable, dump: TurnDump | null, asFrame: (value: T) => ProtocolFrame): AsyncIterable; -export function recordStream( - stream: AsyncIterable, - dump: TurnDump | null, - asFrame: (value: T) => ProtocolFrame = value => value as ProtocolFrame, -): AsyncIterable { - // No recording configured hands the same iterable back, so a record shows no step where - // nothing happened and the stream is not wrapped for nobody. - if (dump === null) return stream; - - const recording = dump.openStream(); - return { - ...recording.fact, - [Symbol.asyncIterator]: () => (async function* () { - for await (const value of stream) { - recording.frame(asFrame(value)); - yield value; - } - // Reached only where the source ran out on its own, which is what makes the record of - // this stream complete. A reader that stopped early never gets here. - recording.end(); - })(), - }; -} - -/** - * The reference a value carries to the stream the record holds, for a wrapper to carry across. - * - * A stream is framed again on its way out — protocol frames become SSE — and what the client - * is handed is a different object over the same frames. Carrying the reference onto it is what - * lets the fact that produced the stream and the fact that framed it point at one record. - */ -export const streamReferenceOf = (value: unknown): StreamFact | Record => - isStreamFact(value) ? { ...value } : {}; From cfc23e87ba5f56b0c1aa4b7981f257b2cf30ebc7 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 12:07:16 +0800 Subject: [PATCH 094/156] feat(openai-responses): make a shim's search a run of its own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The other half of ruling 2-and-6's sub-request clause. The planning stays with the turn — what a call resolves to is the turn's decision — and what runs the backend is a run of its own, with its own prologue, its own settlement and its own record. A search settles differently from an image, and the difference is real rather than an omission. What the search backend charges is accounted per api key in units no model prices, so this run bills no entity: the row it writes names none, which is the settlement stage's own statement that a run which measured rather than generated still writes. And no `PerformanceOperation` names search, so there is no sample for the performance half to land on. What the run buys is the record — a search that ran inside a turn is legible as its own. Which is what the test asserts, because it is the only observable: one stored record on `/alpha/search`, holding the two stages the sub-chain composes. Verified by mutation — calling the resolver directly again leaves zero records. --- .../server-tools/shim_test.ts | 35 ++++++++ .../server-tools/search-sub-request.ts | 84 +++++++++++++++++++ .../server-tools/web-search.ts | 6 +- 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts index 333eba38d9..bd8e759e6e 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/shim_test.ts @@ -27,8 +27,12 @@ import type { WebSearchProviderRequest, WebSearchProviderResult, } from '../../../../../src/data-plane/tools/web-search/types.ts'; +import { initDumpBroker, initDumpStore } from '../../../../../src/dump/registry.ts'; +import { openRunDump } from '../../../../../src/dump/run-sink.ts'; import { getRepo, initRepo } from '../../../../../src/repo/index.ts'; +import { eventsOf, installDumpStubs } from '../../../../dump/test-fixtures.ts'; import { InMemoryRepo } from '../../../../repo/memory.ts'; +import { flushBackground } from '../../../../test-utils/background-tracker.ts'; import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; import { eventFrame } from '@floway-dev/protocols/common'; import type { BillableUsage, ProtocolFrame } from '@floway-dev/protocols/common'; @@ -6282,3 +6286,34 @@ test('consumeTurn forwards an event carrying no output_index instead of dropping assert(forwarded.type === 'event'); assertEquals((forwarded.event as unknown as { detail: string }).detail, 'carried through'); }); + +// ── A search is a run of its own ────────────────────────────────────────── + +// Ruling 2-and-6: a server tool's backend call is not part of the turn that asked for it. What +// makes that observable is the record — a search bills no entity and names no performance +// operation, so the run is the whole of what it buys, and its stages number from 1 like any +// run's, which is why the record has to be its own rather than merged into the turn's. +test('a search runs with its own prologue and its own record', async () => { + const { backend: _backend } = makeStubDeps(); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const turn = openRunDump( + { id: 'k1', dumpRetentionSeconds: 3600 } as never, + { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, + promise => { void promise; }, + ); + assert(turn !== null); + const inv = makeInvocation({ payload: { tools: [{ type: 'web_search' }] } }); + const script = scriptedRun([searchCallTurn(0, 'call_1', 'floway'), messageTurn('done', 0)]); + + await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, mockChatGatewayCtx({ apiKeyId: 'k1', wantsStream: true, dump: turn }), script.run); + await flushBackground(); + + // One record, and it is the search's rather than the turn's: the turn's own is finalized by + // the seam that served it, which no test harness here reaches. + assertEquals(stubs.stored.length, 1); + assertEquals(stubs.stored[0]!.record.meta.path, '/alpha/search'); + const entered = eventsOf(stubs.stored[0]!.record) + .filter(event => event.type === 'stage.entered') + .map(event => event.name); + assertEquals(entered, ['writeSettlement', 'runWebSearchCall']); +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts new file mode 100644 index 0000000000..f8cd50c304 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts @@ -0,0 +1,84 @@ +// The search a shim runs, as a run of its own. +// +// Ruling 2-and-6: a server tool's backend call is not part of the turn that asked for it — it is +// an independent run, with its own prologue, its own settlement and its own record. Its stages +// number from 1 like any run's, which is why the record has to be its own: events landing in the +// parent's would collide with the parent's ids and read as one turn entering a stage twice. +// +// A search settles differently from an image, and the difference is real rather than an +// omission. What the search backend charges is accounted per api key in units no model prices, +// so this run bills no entity — the row it writes names none, which is the settlement stage's +// own statement that a run which measured rather than generated still writes. And no +// `PerformanceOperation` names search, so there is no sample for the performance half to land on. +// What the run buys, then, is the record: a search that ran inside a turn is legible as its own. + +import type { GatewayFacts } from '../../../pipeline/facts.ts'; +import { prologueFor } from '../../../pipeline/serve.ts'; +import type { GatewayServices } from '../../../pipeline/services.ts'; +import { writeSettlement } from '../../../pipeline/settlement.ts'; +import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; +import type { WebSearchCallIR } from '../../../tools/web-search/operations.ts'; +import { compose, defineStage, move, run, type Pipeline } from '@floway-dev/pipeline'; + +/** What one search call is, and what it came to. */ +export interface WebSearchSubRequestFacts extends GatewayFacts { + /** The call, already planned: the operations to run and nothing about the turn that asked. */ + 'request.webSearch.call': () => Promise; + /** What the backend answered, in the shape the hosted-tool item is built from. */ + 'response.webSearch.ir': WebSearchCallIR; +} + +type W = { [P in K]: WebSearchSubRequestFacts[P] }; + +const runWebSearchCall = defineStage< + W<'request.webSearch.call'>, + W<'response.webSearch.ir'> & { 'response.usage.billable': readonly never[] }, + GatewayServices +>({ + name: 'runWebSearchCall', + return: { provides: ['response.webSearch.ir', 'response.usage.billable'] }, + execute: async facts => move({ + ...facts, + 'response.webSearch.ir': await facts['request.webSearch.call'](), + // No model was called, so there is no entity to bill: what the backend charges is accounted + // per api key by the operations as they run. + 'response.usage.billable': [], + }) as never, +}); + +export const webSearchSubRequestPipeline: Pipeline, W<'response.webSearch.ir'>> = + compose('webSearchSubRequest', [ + writeSettlement(() => false), + runWebSearchCall, + ]); + +/** + * The run a shim's search is. + * + * Its context is the parent's, with the three things a separate run owes itself: a start time of + * its own, an attempt slot of its own — so the outer turn's upstream stamp survives — and a + * record of its own. + */ +export const runWebSearchSubRequest = async ( + parent: GatewayCtx, + call: WebSearchSubRequestFacts['request.webSearch.call'], +): Promise => { + const dump = parent.dump?.openSubRequest({ method: 'POST', path: '/alpha/search' }) ?? null; + const gateway: GatewayCtx = { + ...parent, + requestStartedAt: Date.now(), + attempt: { firstOutputTokenAt: null, upstreamCallStartedAt: null, telemetry: undefined }, + dump, + }; + const prologue = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }, dump); + + const { facts, drain } = await run( + webSearchSubRequestPipeline, + move({ 'request.webSearch.call': call }) as never, + prologue.services as never, + ); + // Nothing streams out of a search, so the run is over the moment it answers. + await drain(); + dump?.finalize(200, 0); + return facts['response.webSearch.ir']; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/web-search.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/web-search.ts index 95b921e5eb..2f5e14e702 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/web-search.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/web-search.ts @@ -1,3 +1,4 @@ +import { runWebSearchSubRequest } from './search-sub-request.ts'; import { type ServerToolLoopState, type ServerToolOutputItem, type ServerToolRegistration } from './shim.ts'; import { shortId } from '../../../../shared/short-id.ts'; import { truncatePreservingCodePoints } from '../../../shared/text.ts'; @@ -788,8 +789,11 @@ export const webSearchServerTool: ServerToolRegistration = async (invocation, ga { type: 'response.web_search_call.in_progress' }, { type: 'response.web_search_call.searching' }, ], + // Its own run: its own prologue, its own settlement, its own record. The + // planning above stays with the turn — what a call resolves to is the turn's + // decision — and what runs the backend is a run of its own. run: async function* run() { - const ir = await slot.resolve(); + const ir = await runWebSearchSubRequest(gatewayCtx, slot.resolve); // `results` is gated on the client's `include` // opt-in to match native OpenAI Responses' default wire // shape; the IR keeps them either way for the From 7f206984bf6dbb4f07a306ef7a38d67ccc300c63 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 12:46:17 +0800 Subject: [PATCH 095/156] refactor(affinity): rebuild a candidate's payload rather than clone the conversation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruling 2: nothing in the pipeline is written into, and nothing in it needs a clone. The three affinity materializers were the last place that broke both halves at once — each deep-cloned the record's payload per candidate and then wrote into the copy, which is what `delete replacement.signature` and `content.parts = …` were for. Cloning was how they reached a value they were allowed to edit, and freezing the record is what makes that unnecessary rather than merely wasteful: the payload they are handed cannot be edited, so the only thing left to do is return a new one. Rebuilt identity-preservingly, what one candidate is owed differs from what the next is by the handful of objects a projection actually touched, and a message no projection touched is the same object — instead of a copy of the whole conversation per candidate, on a surface that runs once per failover candidate. The three helpers this needs are the ones the Copilot boundary rules already had, so they move to `@floway-dev/protocols/common` and the provider's private copy goes. `withIndexesChanged` is new: replace-or-drop by index, which is what both the block filter and the part filter were open-coding. --- .../anthropic-messages/affinity/ingress.ts | 29 ++++++----- .../affinity/ingress.ts | 31 +++++++----- .../affinity/ingress.ts | 19 ++++--- packages/protocols/src/common/index.ts | 1 + packages/protocols/src/common/rebuild.ts | 49 +++++++++++++++++++ .../strip-cache-control-extensions.ts | 2 +- .../src/interceptors/shared/rebuild.ts | 38 -------------- 7 files changed, 97 insertions(+), 72 deletions(-) create mode 100644 packages/protocols/src/common/rebuild.ts delete mode 100644 packages/provider-copilot/src/interceptors/shared/rebuild.ts diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/affinity/ingress.ts index e939b02edc..a267e49eae 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/affinity/ingress.ts @@ -1,5 +1,6 @@ import { type AffinityCodec, type AffinityRequestAnalysis, type DecodedAffinityBlob, defineAffinityRequest, projectOptionalAffinityBlob } from '../../shared/affinity/index.ts'; import type { AnthropicMessagesAssistantContentBlock, AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { withIndexesChanged, withKeysChanged } from '@floway-dev/protocols/common'; interface AnthropicMessagesBlobLocation { readonly messageIndex: number; @@ -29,20 +30,22 @@ export const analyzeAnthropicMessagesAffinity = async ( return { kind: 'accepted', degrades: projections.some(item => item.projection.kind === 'remove' && item.projection.degrades), + // Rebuilt rather than cloned: the payload is the record's, so it is frozen, and a message + // no projection touches rides through by identity. What one candidate is owed differs from + // what the next is by a handful of objects, not by a copy of the conversation. materialize: () => { - const candidatePayload = structuredClone(payload); const byMessage = Map.groupBy(projections, item => item.location.messageIndex); const emptiedByAffinity = new Set(); + const rewritten = new Map(); for (const [messageIndex, messageProjections] of byMessage) { - const message = candidatePayload.messages[messageIndex] as { role: 'assistant'; content: AnthropicMessagesAssistantContentBlock[] }; + const message = payload.messages[messageIndex] as { role: 'assistant'; content: AnthropicMessagesAssistantContentBlock[] }; const replacements = new Map(); for (const { location, projection } of messageProjections) { const block = message.content[location.blockIndex]; if (location.kind === 'thinking') { - const replacement = { ...block } as Extract; - if (projection.kind === 'preserve') replacement.signature = projection.value; - else if (projection.kind === 'remove') delete replacement.signature; - replacements.set(location.blockIndex, replacement); + replacements.set(location.blockIndex, withKeysChanged(block, { + signature: projection.kind === 'preserve' ? projection.value : undefined, + })); } else { replacements.set( location.blockIndex, @@ -52,14 +55,14 @@ export const analyzeAnthropicMessagesAffinity = async ( ); } } - message.content = message.content.flatMap((block, blockIndex) => { - const replacement = replacements.get(blockIndex); - return replacement === undefined ? [block] : replacement === null ? [] : [replacement]; - }); - if (message.content.length === 0) emptiedByAffinity.add(messageIndex); + const content = withIndexesChanged(message.content, replacements); + if (content.length === 0) emptiedByAffinity.add(messageIndex); + rewritten.set(messageIndex, withKeysChanged(message, { content }) as AnthropicMessagesPayload['messages'][number]); } - candidatePayload.messages = candidatePayload.messages.filter((_message, messageIndex) => !emptiedByAffinity.has(messageIndex)); - return candidatePayload; + const messages = payload.messages + .map((message, messageIndex) => rewritten.get(messageIndex) ?? message) + .filter((_message, messageIndex) => !emptiedByAffinity.has(messageIndex)); + return withKeysChanged(payload, { messages }); }, }; }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/ingress.ts index 31b37c1a49..3441ba47c2 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/ingress.ts @@ -1,5 +1,6 @@ import { type AffinityCodec, type AffinityRequestAnalysis, type DecodedAffinityBlob, defineAffinityRequest, projectOptionalAffinityBlob } from '../../shared/affinity/index.ts'; -import type { GeminiGenerateContentPart, GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; +import { withIndexesChanged, withKeysChanged } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentContent, GeminiGenerateContentPart, GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; interface GeminiGenerateContentBlobLocation { readonly contentIndex: number; @@ -30,13 +31,17 @@ export const analyzeGeminiGenerateContentAffinity = async ( return { kind: 'accepted', degrades: projections.some(item => item.projection.kind === 'remove' && item.projection.degrades), + // Rebuilt rather than cloned: the payload is the record's, so it is frozen, and a content + // no projection touches rides through by identity. What one candidate is owed differs from + // what the next is by a handful of objects, not by a copy of the conversation. materialize: () => { - const candidatePayload = structuredClone(payload); - if (candidatePayload.contents === undefined) return candidatePayload; + const contents = payload.contents; + if (contents === undefined) return payload; const byContent = Map.groupBy(projections, item => item.location.contentIndex); const emptiedByAffinity = new Set(); + const rewritten = new Map(); for (const [contentIndex, contentProjections] of byContent) { - const content = candidatePayload.contents[contentIndex]; + const content = contents[contentIndex]; const replacements = new Map(); for (const { location, projection } of contentProjections) { const part = content.parts[location.partIndex]; @@ -44,19 +49,19 @@ export const analyzeGeminiGenerateContentAffinity = async ( if (projection.kind === 'preserve') { replacements.set(location.partIndex, { ...part, thoughtSignature: projection.value }); } else { - const replacement = { ...part }; - delete replacement.thoughtSignature; + const replacement = withKeysChanged(part, { thoughtSignature: undefined }); replacements.set(location.partIndex, hasPartContent(replacement) ? replacement : null); } } - content.parts = content.parts.flatMap((part, partIndex) => { - const replacement = replacements.get(partIndex); - return replacement === undefined ? [part] : replacement === null ? [] : [replacement]; - }); - if (content.parts.length === 0) emptiedByAffinity.add(contentIndex); + const parts = withIndexesChanged(content.parts, replacements); + if (parts.length === 0) emptiedByAffinity.add(contentIndex); + rewritten.set(contentIndex, withKeysChanged(content, { parts })); } - candidatePayload.contents = candidatePayload.contents.filter((_content, contentIndex) => !emptiedByAffinity.has(contentIndex)); - return candidatePayload; + return withKeysChanged(payload, { + contents: contents + .map((content, contentIndex) => rewritten.get(contentIndex) ?? content) + .filter((_content, contentIndex) => !emptiedByAffinity.has(contentIndex)), + }); }, }; }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/affinity/ingress.ts index a2817d48cd..1ff7b16fc7 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/affinity/ingress.ts @@ -1,4 +1,5 @@ import { type AffinityCodec, type AffinityRequestAnalysis, type DecodedAffinityBlob, defineAffinityRequest, projectOptionalAffinityBlob } from '../../shared/affinity/index.ts'; +import { withKeysChanged } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; export const analyzeOpenAIChatCompletionsAffinity = async ( @@ -16,14 +17,18 @@ export const analyzeOpenAIChatCompletionsAffinity = async ( return { kind: 'accepted', degrades: projections.some(item => item.projection.kind === 'remove' && item.projection.degrades), + // Rebuilt rather than cloned: the payload is the record's, so it is frozen, and a message + // no projection touches rides through by identity. What one candidate is owed differs from + // what the next is by a handful of objects, not by a copy of the conversation. materialize: () => { - const candidatePayload = structuredClone(payload); - for (const { index, projection } of projections) { - const message = candidatePayload.messages[index]; - if (projection.kind === 'preserve') message.reasoning_opaque = projection.value; - else if (projection.kind === 'remove') delete message.reasoning_opaque; - } - return candidatePayload; + const rewritten = new Map(projections.map(({ index, projection }) => [ + index, + withKeysChanged(payload.messages[index], { + reasoning_opaque: projection.kind === 'preserve' ? projection.value : undefined, + }), + ])); + const messages = payload.messages.map((message, index) => rewritten.get(index) ?? message); + return withKeysChanged(payload, { messages }); }, }; }); diff --git a/packages/protocols/src/common/index.ts b/packages/protocols/src/common/index.ts index 6248b39afa..cc622541fa 100644 --- a/packages/protocols/src/common/index.ts +++ b/packages/protocols/src/common/index.ts @@ -16,3 +16,4 @@ export * from './parse-events.ts'; export { renderErrorEnvelope, renderProtocolError, upstreamErrorMessage } from './error-envelope.ts'; export { isJsonObject, type JsonObject } from './json.ts'; export { captureExtras } from './reassemble-extras.ts'; +export { mapKeepingIdentity, withIndexesChanged, withKeysChanged } from './rebuild.ts'; diff --git a/packages/protocols/src/common/rebuild.ts b/packages/protocols/src/common/rebuild.ts new file mode 100644 index 0000000000..5fda71756a --- /dev/null +++ b/packages/protocols/src/common/rebuild.ts @@ -0,0 +1,49 @@ +// Rebuilding a value instead of writing into one. +// +// A record's values are frozen — a fact that could be edited after the event is not a fact — so +// anything that shapes a payload it was handed returns a new one. What these two add is the +// discipline one level down, without paying for a copy of the whole tree: a rewrite that changed +// nothing hands the same object back, so an untouched message or block rides through by identity +// and only the path that actually changed is new. That is the same rule the record itself is +// folded by, and it is what keeps a long conversation costing a handful of objects rather than a +// clone per candidate. + +/** Applies `rewrite` to every item, and hands back the original array when none of them + * changed — so "nothing happened here" is the same test at every level. */ +export const mapKeepingIdentity = (items: readonly T[], rewrite: (item: T) => T): readonly T[] => { + let changed = false; + const next = items.map(item => { + const rewritten = rewrite(item); + if (rewritten !== item) changed = true; + return rewritten; + }); + return changed ? next : items; +}; + +/** The object form: rebuild with `patch` applied, or hand back the original when the patch is + * empty. `undefined` in the patch removes the key, which is what a rest-destructuring would + * do and what `delete` used to do in place. */ +export const withKeysChanged = (source: T, patch: Readonly>): T => { + const keys = Object.keys(patch); + if (keys.length === 0) return source; + const next: Record = { ...(source as Record) }; + for (const key of keys) { + if (patch[key] === undefined) delete next[key]; + else next[key] = patch[key]; + } + return next as T; +}; + +/** An array with one index replaced, or removed where the replacement is `null`. Every other + * element rides through by identity, and an index nobody replaced leaves the array itself + * untouched. */ +export const withIndexesChanged = (items: readonly T[], replacements: ReadonlyMap): readonly T[] => { + if (replacements.size === 0) return items; + const next: T[] = []; + items.forEach((item, index) => { + if (!replacements.has(index)) { next.push(item); return; } + const replacement = replacements.get(index)!; + if (replacement !== null) next.push(replacement); + }); + return next; +}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts index 69901a4bdc..241d8bf68c 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts @@ -1,5 +1,5 @@ import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import { mapKeepingIdentity, withKeysChanged } from '../shared/rebuild.ts'; +import { mapKeepingIdentity, withKeysChanged } from '@floway-dev/protocols/common'; /** * Two `cache_control` sub-fields are beta extensions to the base diff --git a/packages/provider-copilot/src/interceptors/shared/rebuild.ts b/packages/provider-copilot/src/interceptors/shared/rebuild.ts deleted file mode 100644 index db6d4e11a7..0000000000 --- a/packages/provider-copilot/src/interceptors/shared/rebuild.ts +++ /dev/null @@ -1,38 +0,0 @@ -// Rebuilding a payload instead of writing into one. -// -// A boundary interceptor is handed the record's own value, and the record is frozen — a fact -// that could be edited after the event is not a fact. So a rule that shapes the outgoing -// payload returns a new one rather than reaching into the old, and every provider here -// already does that at the top level: `ctx.payload = { ...ctx.payload, … }`. -// -// What these two add is the same discipline one level down, without paying for a copy of the -// whole tree. A rewrite that changed nothing hands the same object back, so an untouched -// message or block rides through by identity and only the path that actually changed is -// rebuilt — which is what keeps a long conversation costing a handful of objects rather than -// a full clone, and is the same rule the record itself is folded by. - -/** Applies `rewrite` to every item, and hands back the original array when none of them - * changed — so "nothing happened here" is the same test at every level. */ -export const mapKeepingIdentity = (items: readonly T[], rewrite: (item: T) => T): readonly T[] => { - let changed = false; - const next = items.map(item => { - const rewritten = rewrite(item); - if (rewritten !== item) changed = true; - return rewritten; - }); - return changed ? next : items; -}; - -/** The object form: rebuild with `patch` applied, or hand back the original when the patch is - * empty. `undefined` in the patch removes the key, which is what a rest-destructuring would - * do and what `delete` used to do in place. */ -export const withKeysChanged = (source: T, patch: Readonly>): T => { - const keys = Object.keys(patch); - if (keys.length === 0) return source; - const next: Record = { ...(source as Record) }; - for (const key of keys) { - if (patch[key] === undefined) delete next[key]; - else next[key] = patch[key]; - } - return next as T; -}; From 3cb711cb59cee153d23d7f8db41a724c4dc3f989 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 12:46:39 +0800 Subject: [PATCH 096/156] refactor(chat): say rules where the chain runs rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration is over, so the vocabulary that described it should not outlive it. Four comments and six test titles still called the chat rules interceptors, and `stage.ts` still justified the pass-through direction by counting calls in the surface it replaced — a census that reads as current fact and is not one. No behaviour here; the names are the deliverable. --- .../__tests__/data-plane/chat-rules_test.ts | 20 +++++++++---------- .../openai-completions/http_test.ts | 2 +- .../openai-completions/usage_test.ts | 2 +- .../__tests__/test-utils/gateway-ctx.ts | 4 ++-- packages/pipeline/src/stage.ts | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat-rules_test.ts b/packages/gateway/__tests__/data-plane/chat-rules_test.ts index a4dba3a1eb..6264d00110 100644 --- a/packages/gateway/__tests__/data-plane/chat-rules_test.ts +++ b/packages/gateway/__tests__/data-plane/chat-rules_test.ts @@ -1,6 +1,6 @@ -// The chat interceptors, migrated. Two properties matter and neither is about the rewrites -// themselves — those are transcribed from code that already worked. What is new is that a -// stage returns a record instead of assigning to a context, and that a rewrite which changes +// The chat rules. Two properties matter and neither is about the rewrites themselves — what +// each rule does to a payload is settled by the protocol it speaks. What matters here is that +// a stage returns a record instead of assigning to a context, and that a rewrite which changes // nothing changes nothing by identity. A rule that speaks about the response direction has a // third: what it read on the way down has to reach the way back, which is the only thing its // closure is for. @@ -44,7 +44,7 @@ import type { OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesS const attemptWith = (...flags: string[]): AttemptSelector => ({ upstreamId: 'u', modelId: 'm', flags }); /** - * Runs a family's whole interceptor array, in the order its chain runs it, between stages + * Runs a family's whole rule array, in the order its chain runs it, between stages * that declare what its real neighbours declare: an edge that needs the answer, the headers * and the billed set on the way up, and an ending that answers with all three. * @@ -120,7 +120,7 @@ const anthropicMessagesPayload = (messages: unknown[], rest: Record = {}): OpenAIResponsesPayload => ({ model: 'm', input, ...rest }) as unknown as OpenAIResponsesPayload; -describe('the OpenAI Chat Completions interceptors, as stages', () => { +describe('the OpenAI Chat Completions rules', () => { it('rewrites the roles its flags name, in the settled order', async () => { const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ @@ -426,7 +426,7 @@ const assistantReplay = (reasoning: Record): OpenAIChatCompleti const assistantOf = (payload: unknown): Record => (payload as OpenAIChatCompletionsPayload).messages[1] as unknown as Record; -describe('the OpenAI Chat Completions vendor dialects, as stages', () => { +describe('the OpenAI Chat Completions vendor dialects', () => { it('projects the canonical assistant reasoning onto the one field DeepSeek reads', async () => { const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload(assistantReplay({ @@ -650,7 +650,7 @@ describe('the OpenAI Chat Completions vendor dialects, as stages', () => { }); }); -describe('the Anthropic Messages interceptors, as stages', () => { +describe('the Anthropic Messages rules', () => { it('turns every inline system message into a user one', async () => { const { down } = await runStage(applyRoleCompatibilityToAnthropicMessages, 'response.chat.anthropicMessages', { 'request.chat.anthropicMessages': anthropicMessagesPayload([ @@ -751,7 +751,7 @@ describe('the Anthropic Messages interceptors, as stages', () => { }); }); -describe('the Gemini generateContent interceptors, as stages', () => { +describe('the Gemini generateContent rules', () => { it('drops the part fields no translation carries, and a part they were all of', async () => { const { down } = await runStage(stripUnsupportedPartFieldsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { 'request.chat.geminiGenerateContent': { @@ -859,7 +859,7 @@ describe('the Gemini generateContent interceptors, as stages', () => { }); }); -describe('the OpenAI Responses interceptors, as stages', () => { +describe('the OpenAI Responses rules', () => { const rolesOf = (down: Record): unknown[] => ((down['request.chat.openaiResponses'] as OpenAIResponsesPayload).input as OpenAIResponsesInputItem[]) .map(item => (item as { role?: unknown }).role); @@ -1034,7 +1034,7 @@ const usageOf = (frame: ProtocolFrame): Record { +describe('a family\'s rule array, in the order its chain runs it', () => { it('assembles and runs the Anthropic Messages array', async () => { const { down } = await runChain('response.chat.anthropicMessages', [ stripBillingAttributionFromAnthropicMessages, diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts index bc71c9bd7d..ea61d21945 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts @@ -268,7 +268,7 @@ test('/v1/completions handler also serves the unversioned /completions path', as // Dual-path coverage: usage + performance + dump telemetry must land // equivalently on both the non-streaming JSON and the streaming SSE // branches. The handler does not force `stream: true` upstream the way -// openai-chat-completions does (no interceptor framework to feed), so the two +// openai-chat-completions does (no chat chain to feed), so the two // paths really do exercise different scaffold branches and the assertions // here keep them honest. test('/v1/completions non-streaming records usage row, performance neutral row (text_completion operation, no TTFT/TPOT), and a bytes-body dump record', async () => { diff --git a/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts index daf3ef7075..b5559648da 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts @@ -87,7 +87,7 @@ test('tokenUsageFromOpenAICompletionsUsage runs serviceTier through billableServ test('tokenUsageFromOpenAICompletionsUsage folds the cache counts back in on the totals alone', () => { // The same accounting the chat targets normalize on the usage chunk: this - // endpoint has no interceptor chain, so the fold happens on the billing read. + // endpoint runs no chat rules, so the fold happens on the billing read. assertEquals( tokenUsageFromOpenAICompletionsUsage( { prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_tokens_details: { cached_tokens: 13312 } }, diff --git a/packages/gateway/__tests__/test-utils/gateway-ctx.ts b/packages/gateway/__tests__/test-utils/gateway-ctx.ts index 4ba8e4085d..f257e70544 100644 --- a/packages/gateway/__tests__/test-utils/gateway-ctx.ts +++ b/packages/gateway/__tests__/test-utils/gateway-ctx.ts @@ -4,8 +4,8 @@ import type { ChatGatewayCtx } from '../../src/data-plane/chat/shared/gateway-ct import type { GatewayCtx } from '../../src/data-plane/shared/gateway-ctx.ts'; import { stubModelCandidate } from '@floway-dev/test-utils'; -// Shared minimal GatewayCtx for tests that exercise serve / respond / -// interceptor code in isolation. Defaults satisfy every required field; pass +// Shared minimal GatewayCtx for tests that exercise serve / respond / stage +// code in isolation. Defaults satisfy every required field; pass // `overrides` to nudge what each test cares about. Callers that need a // downstream abort controller should construct one and spread // `{ abortSignal: controller.signal, downstreamAbortController: controller }` diff --git a/packages/pipeline/src/stage.ts b/packages/pipeline/src/stage.ts index bf406bef5f..0e34134fa6 100644 --- a/packages/pipeline/src/stage.ts +++ b/packages/pipeline/src/stage.ts @@ -205,8 +205,8 @@ export function defineStage(s: object): Stage { // `transform` fills the `execute` slot. Two layers: `open` holds the per-invocation state // that spans both directions, and each direction is one function returning the record it // hands on. Either side may be omitted, and an omitted one passes through unchanged — -// which is the point, because today's interceptors call the continuation 107 times across -// 68 files and 50 of those calls are a bare `return run();`. +// which is the point, because most rules shape one direction and would otherwise spend a +// line handing the other one back untouched. // // A stream, a value and a failure sit at one key, so telling them apart is reading a // value, and the stage does that where it needs to. There is no outcome-shape dispatch. From fea5ab6e949885dfe46a8dfc85ba2dd01411421e Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 13:21:25 +0800 Subject: [PATCH 097/156] fix(gemini-generate-content): send a minted envelope with the status it names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruling 4: the coercion the replaced surface applied is behavior to keep, not a rough edge the migration was free to drop. A Google-RPC envelope states its code numerically in `error.code` and by name in `error.status`, and this protocol has a name for nine codes — everything else is `INTERNAL`, which belongs to 500 alone. So a 402 refusal minted from an upstream that spoke another protocol was answered as `code: 402` beside `status: "INTERNAL"`, sent as a 402: three statements, two of them contradicting the third. `main` answered 500 (`gemini/respond.ts:132`); nothing tested it, which is how the migration lost it. What let it be lost is that the body and the status were separate decisions — `renderFailure` returned an envelope and every edge then wrote `answer.status` beside it, free to disagree. They are now one: `renderFailure` hands back both, and only the third tier — the envelope this gateway words itself — may answer with a status other than the failure's. A forwarded upstream body and a refusal's own envelope keep theirs, because neither is this gateway's to reword. That makes the class structural rather than remembered. `mintedAs` covers the protocols whose envelope says nothing about its status, Gemini generateContent mints both together, and the name-for-a-status function is private to the module that mints — so no caller holds the half that can disagree. The three open-coded copies of the envelope (two edges and the models endpoint) go through it too. --- .../routes-gemini-generate-content_test.ts | 18 +++--- .../pipeline/failure-render_test.ts | 62 ++++++++++++++++--- .../chat/anthropic-messages/count-tokens.ts | 7 ++- .../chat/anthropic-messages/pipeline.ts | 7 ++- .../gemini-generate-content/count-tokens.ts | 7 ++- .../chat/gemini-generate-content/errors.ts | 46 +++++++++++--- .../chat/gemini-generate-content/http.ts | 6 +- .../chat/gemini-generate-content/pipeline.ts | 14 +++-- .../chat/openai-chat-completions/pipeline.ts | 10 ++- .../chat/openai-responses/compact.ts | 10 ++- .../chat/openai-responses/pipeline.ts | 10 ++- .../gateway/src/data-plane/models/gemini.ts | 16 ++--- .../gateway/src/data-plane/pipeline/facts.ts | 30 ++++++++- 13 files changed, 170 insertions(+), 73 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts index e8b9b41010..c0afdb8c69 100644 --- a/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts @@ -130,9 +130,11 @@ test('an upstream refusal reaches the client in this protocol-s own envelope', a async () => await post(apiKey.key, 'gpt-route:generateContent'), ); - // The status the upstream sent, with none of the coercion a Google-RPC envelope would force - // onto the codes it cannot express. - expect(response.status).toBe(402); + // Not the 402 the upstream sent. This envelope is minted rather than forwarded, and it + // states its code inside the body as well as naming it — a code this protocol has no name + // for reads `INTERNAL`, which belongs to 500. Answering 402 would put `code: 402` beside + // `status: "INTERNAL"`, so the status the envelope can express is the one that is sent. + expect(response.status).toBe(500); // Gemini generateContent has no wire of its own, so every refusal that gets here was written by whatever // protocol the candidate was dialled on — here OpenAI's. Handing that object on would answer @@ -140,9 +142,9 @@ test('an upstream refusal reaches the client in this protocol-s own envelope', a // an OpenAI envelope has none, while `error.code` would arrive as a string where a number // belongs. So the upstream's words become the message and this protocol writes the shape. const body = await response.json() as { error: { code: number; message: string; status: string } }; - expect(body.error.code).toBe(402); - // The Google-RPC name for the status, which is the field an SDK branches on. A code this - // envelope cannot express reads `INTERNAL`, and 402 is one of them. + expect(body.error.code).toBe(500); + // The Google-RPC name for the status, which is the field an SDK branches on — and the one + // the code above agrees with, because the two are decided together. expect(body.error.status).toBe('INTERNAL'); expect(body.error.message).toContain('copilot said no'); }); @@ -160,10 +162,10 @@ test('a refusal on a turn that asked to stream is a body, not an opened stream', async () => await post(apiKey.key, 'gpt-route:streamGenerateContent'), ); - expect(response.status).toBe(402); + expect(response.status).toBe(500); expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); const body = await response.json() as { error: { code: number; message: string; status: string } }; - expect(body.error.code).toBe(402); + expect(body.error.code).toBe(500); expect(body.error.message).toContain('copilot said no'); }); diff --git a/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts b/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts index cec6d12767..5cbfffcfb6 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts @@ -1,22 +1,27 @@ -// What a client is sent when a turn failed. +// What a client is sent when a turn failed, and the status it is sent with. // // The three tiers are easy to collapse by accident — an edge that reaches for the protocol's // default first will bury both an upstream's own words and a refusal's own envelope, and each // family renders its own errors, so the rule is worth holding in one place and pinning here. +// +// The status travels with the body because only the third tier may change it, and one protocol +// must: a Google-RPC envelope states its code inside the body, so an edge free to pick a status +// separately can state two different things about one failure. import { describe, it } from 'vitest'; -import { renderFailure } from '../../../src/data-plane/pipeline/facts.ts'; +import { mintGeminiGenerateContentFailure } from '../../../src/data-plane/chat/gemini-generate-content/errors.ts'; +import { mintedAs, renderFailure } from '../../../src/data-plane/pipeline/facts.ts'; import { assertEquals } from '@floway-dev/test-utils'; -const protocolDefault = () => ({ error: { message: 'rendered from the status', type: 'api_error' } }); +const protocolDefault = mintedAs(() => ({ error: { message: 'rendered from the status', type: 'api_error' } })); describe('renderFailure', () => { it('forwards the body the upstream sent', () => { const upstream = { error: { message: 'model is overloaded', type: 'overloaded_error' } }; assertEquals( renderFailure({ status: 529, message: 'upstream refused', body: upstream }, protocolDefault), - upstream, + { body: upstream, status: 529 }, ); }); @@ -24,7 +29,7 @@ describe('renderFailure', () => { const envelope = { error: { message: 'no route', type: 'invalid_request_error', code: 'responses_item_routing_unavailable' } }; assertEquals( renderFailure({ status: 400, message: 'no route', envelope }, protocolDefault), - envelope, + { body: envelope, status: 400 }, ); }); @@ -33,17 +38,58 @@ describe('renderFailure', () => { const envelope = { error: { message: 'no route' } }; assertEquals( renderFailure({ status: 400, message: 'no route', body: upstream, envelope }, protocolDefault), - upstream, + { body: upstream, status: 400 }, ); }); it('falls back to the protocol own rendering when neither is present', () => { - assertEquals(renderFailure({ status: 502, message: 'dial failed' }, protocolDefault), protocolDefault()); + assertEquals( + renderFailure({ status: 502, message: 'dial failed' }, protocolDefault), + { body: { error: { message: 'rendered from the status', type: 'api_error' } }, status: 502 }, + ); }); it('renders the protocol default where the upstream body was not an object', () => { // A gateway that forwarded `"Bad Gateway"` verbatim would answer a JSON protocol with a // bare string, which no client of it can read. - assertEquals(renderFailure({ status: 502, message: 'dial failed', body: 'Bad Gateway' }, protocolDefault), protocolDefault()); + assertEquals( + renderFailure({ status: 502, message: 'dial failed', body: 'Bad Gateway' }, protocolDefault), + { body: { error: { message: 'rendered from the status', type: 'api_error' } }, status: 502 }, + ); + }); +}); + +// Gemini generateContent is the reason the status is the third tier's to decide, so the rule +// is pinned on the protocol that exercises it rather than only on the helper. +describe('the Gemini generateContent third tier', () => { + it('sends a code it has no name for as the one INTERNAL belongs to', () => { + // 418 maps to `INTERNAL`, and `INTERNAL` is 500's name. Sending it as a 418 would answer + // with `code: 418` beside `status: "INTERNAL"` — two statements about one failure. + assertEquals( + renderFailure({ status: 418, message: 'upstream was a teapot' }, mintGeminiGenerateContentFailure), + { body: { error: { code: 500, message: 'upstream was a teapot', status: 'INTERNAL' } }, status: 500 }, + ); + }); + + it('leaves a code it can name alone', () => { + assertEquals( + renderFailure({ status: 429, message: 'slow down' }, mintGeminiGenerateContentFailure), + { body: { error: { code: 429, message: 'slow down', status: 'RESOURCE_EXHAUSTED' } }, status: 429 }, + ); + }); + + it('does not reword an upstream body or a refusal envelope', () => { + // Neither tier is this gateway's to restate, so a 418 survives both ways when someone + // else already answered with it. + const upstream = { error: { code: 418, message: 'upstream said so', status: 'WHATEVER' } }; + assertEquals( + renderFailure({ status: 418, message: 'ignored', body: upstream }, mintGeminiGenerateContentFailure), + { body: upstream, status: 418 }, + ); + const envelope = { error: { code: 418, message: 'we said so', status: 'INVALID_ARGUMENT' } }; + assertEquals( + renderFailure({ status: 418, message: 'ignored', envelope }, mintGeminiGenerateContentFailure), + { body: envelope, status: 418 }, + ); }); }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index 87e0b6baf1..b565f77c5e 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -33,7 +33,7 @@ import type { AnthropicMessagesFacts } from './pipeline.ts'; import { prepareAnthropicMessagesWebSearchShimRequest } from './web-search-shim.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; -import { isFailure, renderFailure } from '../../pipeline/facts.ts'; +import { isFailure, mintedAs, renderFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; @@ -95,11 +95,12 @@ const emitAnthropicMessagesTokenCount = defineStage< const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintedAs(({ status, message }) => renderAnthropicMessagesError(status, message))); return { ...rest, 'response.http.headers': forClient, - 'response.chat.anthropicMessages.rendered': move(renderFailure(answer, () => renderAnthropicMessagesError(answer.status, answer.message))), - 'response.http.status': answer.status, + 'response.chat.anthropicMessages.rendered': move(failure.body), + 'response.http.status': failure.status, }; } return { diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index c35a608393..e74d4caf1c 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -28,7 +28,7 @@ import { runAnthropicMessagesWebSearchTool } from './web-search-tool.ts'; import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; -import { isFailure, renderFailure } from '../../pipeline/facts.ts'; +import { isFailure, mintedAs, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -116,11 +116,12 @@ const emitAnthropicMessages = defineStage< const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintedAs(({ status, message }) => renderAnthropicMessagesError(status, message))); return { ...rest, 'response.http.headers': forClient, - 'response.chat.anthropicMessages.rendered': move(renderFailure(answer, () => renderAnthropicMessagesError(answer.status, answer.message))), - 'response.http.status': answer.status, + 'response.chat.anthropicMessages.rendered': move(failure.body), + 'response.http.status': failure.status, }; } if (answer.kind === 'value') { diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index 5668362080..295f1c9321 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -28,7 +28,7 @@ // stream, and a measurement has none. import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; -import { renderGeminiGenerateContentError } from './errors.ts'; +import { mintGeminiGenerateContentFailure } from './errors.ts'; import type { GeminiGenerateContentFacts } from './pipeline.ts'; import { asJsonObject, readJsonNumber } from '../../../shared/json-helpers.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; @@ -86,11 +86,12 @@ const emitGeminiGenerateContentTokenCount = defineStage< const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintGeminiGenerateContentFailure); return { ...rest, 'response.http.headers': forClient, - 'response.chat.geminiGenerateContent.rendered': move(renderFailure(answer, () => renderGeminiGenerateContentError(answer.status, answer.message))), - 'response.http.status': answer.status, + 'response.chat.geminiGenerateContent.rendered': move(failure.body), + 'response.http.status': failure.status, }; } return { diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts index b807729d8d..49167b5469 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts @@ -1,13 +1,45 @@ // Google RPC Status envelope, used by Gemini's `error` channel everywhere // (HTTP body, SSE-tunnelled error event). -/** What a Gemini generateContent client is sent when a turn produced no content. `error.status` is the - * Google-RPC name and clients read it, so this shape is the protocol's rather than the - * gateway's — an OpenAI-shaped envelope would leave that field undefined. */ -export const renderGeminiGenerateContentError = (status: number, message: string): Record => ({ - error: { code: status, message, status: geminiGenerateContentStatusForHttpStatus(status) }, -}); -export const geminiGenerateContentStatusForHttpStatus = (status: number): string => { +/** + * What a Gemini generateContent client is sent when this gateway is the one refusing: the + * envelope, and the status it is sent with. + * + * `error.status` is the Google-RPC name and clients read it, so the shape is the protocol's + * rather than the gateway's — an OpenAI-shaped envelope would leave that field undefined. + * + * The two are decided together because the envelope states its code as well as naming it, and + * this protocol has a name for only nine codes: everything else is `INTERNAL`, which belongs + * to 500 alone. Minting from some other status would say two different things about one + * failure — `code: 418` beside `status: "INTERNAL"`, sent as a 418 — so a status this protocol + * cannot name is sent as the one that name belongs to. Deciding them apart is what let that + * disagreement exist, which is why nothing here renders an envelope without also saying what + * to send it with. + * + * A forwarded upstream body and a refusal's own envelope keep their status untouched: they + * already say what they mean, and neither is this gateway's to reword. + */ +export const mintGeminiGenerateContentFailure = (failure: { readonly status: number; readonly message: string }): { + readonly body: Record; + readonly status: number; +} => { + const status = geminiGenerateContentStatusForHttpStatus(failure.status) === 'INTERNAL' ? 500 : failure.status; + return { + body: { error: { code: status, message: failure.message, status: geminiGenerateContentStatusForHttpStatus(status) } }, + status, + }; +}; + +/** The same refusal as an HTTP response, for the edges that answer before a pipeline runs. */ +export const geminiGenerateContentErrorResponse = (status: number, message: string): Response => { + const failure = mintGeminiGenerateContentFailure({ status, message }); + return Response.json(failure.body, { status: failure.status }); +}; + +/** The Google-RPC name for an HTTP status. Private, because a caller holding only the name + * can write an envelope whose code contradicts it — which is the whole reason the two are + * minted together above. */ +const geminiGenerateContentStatusForHttpStatus = (status: number): string => { switch (status) { case 400: return 'INVALID_ARGUMENT'; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts index 897f6ec521..1fa59a7504 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts @@ -7,7 +7,7 @@ // it over, and turn what the run answered with into a response. Everything between is stages. import { geminiGenerateContentCountTokensPipeline } from './count-tokens.ts'; -import { renderGeminiGenerateContentError } from './errors.ts'; +import { geminiGenerateContentErrorResponse } from './errors.ts'; import { geminiGenerateContentServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; @@ -30,7 +30,7 @@ interface GeminiGenerateContentModelAction { // SDKs send it). /** A refusal about the route itself. It is answered before a model is resolved, so there is no * attempt to record and nothing for a run to hold beyond the refusal. */ -const unknownAction = (message: string): Response => Response.json(renderGeminiGenerateContentError(404, message), { status: 404 }); +const unknownAction = (message: string): Response => geminiGenerateContentErrorResponse(404, message); const parseGeminiGenerateContentModelAction = (modelAction: string | undefined): GeminiGenerateContentModelAction | Response => { if (!modelAction) return unknownAction('Missing Gemini model action.'); @@ -77,7 +77,7 @@ const readRequest = (bytes: Uint8Array, project: (body: unknown) => T): { typ const refuse = (c: AuthedContext, ingress: Ingress, message: string): Response => { const refused = openPrologue(c, ingress, { wantsStream: false }); refused.gateway.dump?.error('gateway'); - return finalizeGatewayResponse(refused.gateway, Response.json(renderGeminiGenerateContentError(400, message), { status: 400 })); + return finalizeGatewayResponse(refused.gateway, geminiGenerateContentErrorResponse(400, message)); }; const runGeminiGenerateContentGenerate = async (c: AuthedContext, model: string, wantsStream: boolean): Promise => { diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index 47d0b8ca7d..310ce6888d 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -41,13 +41,14 @@ // is in none of them. So the words survive and the shape is this protocol's, which is what // the handoff arranges by dropping a foreign envelope on the way up. // -// The status is still the upstream's, with no coercion of the codes a Google-RPC envelope -// maps to `INTERNAL`. That part is a departure from the replaced surface, which raised such a -// status to 500. +// The status is the upstream's wherever the upstream stated one. Where this gateway mints the +// envelope itself, it is the status the envelope's own name belongs to: a Google-RPC body +// states its code twice, so `mintGeminiGenerateContentFailure` decides the two together and a +// code this protocol cannot name is sent as the 500 that `INTERNAL` means. import { wrapGeminiGenerateContentAffinityEgress } from './affinity/egress.ts'; import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; -import { renderGeminiGenerateContentError } from './errors.ts'; +import { mintGeminiGenerateContentFailure } from './errors.ts'; import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; @@ -136,11 +137,12 @@ const emitGeminiGenerateContent = defineStage< const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintGeminiGenerateContentFailure); return { ...rest, 'response.http.headers': forClient, - 'response.chat.geminiGenerateContent.rendered': move(renderFailure(answer, () => renderGeminiGenerateContentError(answer.status, answer.message))), - 'response.http.status': answer.status, + 'response.chat.geminiGenerateContent.rendered': move(failure.body), + 'response.http.status': failure.status, }; } if (answer.kind === 'value') { diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index 1832304039..ee55bfd15f 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -26,7 +26,7 @@ import { billableUsageFromOpenAIChatCompletionsEvent } from './usage.ts'; import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; -import { isFailure, renderFailure } from '../../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -124,14 +124,12 @@ const emitOpenAIChatCompletions = defineStage< const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); return { ...rest, 'response.http.headers': forClient, - 'response.chat.openaiChatCompletions.rendered': move(renderFailure( - answer, - () => ({ error: { message: answer.message, type: 'api_error' } }), - )), - 'response.http.status': answer.status, + 'response.chat.openaiChatCompletions.rendered': move(failure.body), + 'response.http.status': failure.status, }; } if (answer.kind === 'value') { diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index e9416220d2..b0bfeffaa8 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -64,7 +64,7 @@ import { billableUsageFromOpenAIResponsesResult } from './usage.ts'; import { recordStream } from '../../../dump/run-sink.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { Failure } from '../../pipeline/facts.ts'; -import { isFailure, renderFailure } from '../../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -133,14 +133,12 @@ const emitOpenAIResponsesCompaction = defineStage< const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); return { ...rest, 'response.http.headers': forClient, - 'response.chat.openaiResponses.rendered': move(renderFailure( - answer, - () => ({ error: { message: answer.message, type: 'api_error' } }), - )), - 'response.http.status': answer.status, + 'response.chat.openaiResponses.rendered': move(failure.body), + 'response.http.status': failure.status, }; } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 5d36198f96..6fe1648fc4 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -60,7 +60,7 @@ import { billableUsageFromOpenAIResponsesEvent, billableUsageFromOpenAIResponses import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; import type { AttemptSelector, BillableEntity } from '../../pipeline/facts.ts'; -import { isFailure, renderFailure } from '../../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../../pipeline/facts.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { failover } from '../../pipeline/stages.ts'; @@ -179,14 +179,12 @@ const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, framing: O const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); return { ...rest, 'response.http.headers': forClient, - 'response.chat.openaiResponses.rendered': move(renderFailure( - answer, - () => ({ error: { message: answer.message, type: 'api_error' } }), - )), - 'response.http.status': answer.status, + 'response.chat.openaiResponses.rendered': move(failure.body), + 'response.http.status': failure.status, }; } // A turn the upstream answered with one body rather than a stream — the compaction diff --git a/packages/gateway/src/data-plane/models/gemini.ts b/packages/gateway/src/data-plane/models/gemini.ts index 30e450940d..0cf2c15e5a 100644 --- a/packages/gateway/src/data-plane/models/gemini.ts +++ b/packages/gateway/src/data-plane/models/gemini.ts @@ -7,7 +7,7 @@ import { getRepo } from '../../repo/index.ts'; import type { ModelAliasesRepo } from '../../repo/types.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; -import { geminiGenerateContentStatusForHttpStatus } from '../chat/gemini-generate-content/errors.ts'; +import { geminiGenerateContentErrorResponse } from '../chat/gemini-generate-content/errors.ts'; import { enumerateAddressableModelIds, listedRealModels } from '../shared/listing/addressable.ts'; import { mergeAliasesIntoModels } from '../shared/listing/alias.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; @@ -52,17 +52,11 @@ const toGeminiModel = (model: InternalModel): GeminiModel => { }; }; -const geminiError = (status: number, message: string): Response => - Response.json( - { error: { code: status, message, status: geminiGenerateContentStatusForHttpStatus(status) } }, - { status: status as 400 | 404 | 500 | 502 }, - ); - const geminiModelLoadError = (error: unknown): Response => { if (error instanceof ProviderModelsUnavailableError) { - return geminiError(502, MODEL_LISTING_FAILURE_MESSAGE); + return geminiGenerateContentErrorResponse(502, MODEL_LISTING_FAILURE_MESSAGE); } - return geminiError(502, error instanceof Error ? error.message : String(error)); + return geminiGenerateContentErrorResponse(502, error instanceof Error ? error.message : String(error)); }; // Real chat models plus chat-kind alias entries; collision and dedupe ride @@ -107,13 +101,13 @@ export const serveGeminiModels = async (c: Context): Promise => { export const serveGeminiModelInfo = async (c: Context): Promise => { const rawModelId = c.req.param('modelId'); - if (!rawModelId) return geminiError(404, 'Model not found: '); + if (!rawModelId) return geminiGenerateContentErrorResponse(404, 'Model not found: '); const modelId = rawModelId.replace(/^models\//, ''); try { const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw)); const model = (await loadGeminiModels(effectiveUpstreamIdsFromContext(c), fetcherForUpstream, backgroundSchedulerFromContext(c), getRepo().modelAliases)).find(candidate => candidate.baseModelId === modelId || candidate.name === `models/${modelId}`); - if (!model) return geminiError(404, `Model not found: ${modelId}`); + if (!model) return geminiGenerateContentErrorResponse(404, `Model not found: ${modelId}`); return Response.json(model); } catch (error) { return geminiModelLoadError(error); diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index a2ff294b65..2b408e3a3a 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -10,7 +10,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { Secret, Owned } from '@floway-dev/pipeline'; -import { renderProtocolError, type PricingRuntimeFacts } from '@floway-dev/protocols/common'; +import { isJsonObject, renderErrorEnvelope, type PricingRuntimeFacts } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; /** Everything about an attempt that is data: which upstream, which model row on it, and the @@ -55,6 +55,14 @@ export interface Failure { export const isFailure = (value: unknown): value is Failure => typeof value === 'object' && value !== null && 'status' in value && 'message' in value; +/** What a failure is sent as. The two travel together because for some protocols they are one + * decision: a Google-RPC envelope states its code inside the body, so a status chosen apart + * from the envelope is a second answer to the same question. */ +export interface RenderedFailure { + readonly body: Record; + readonly status: number; +} + /** * What a client is sent for a failure, in three tiers, so no edge has to remember them. * @@ -62,9 +70,25 @@ export const isFailure = (value: unknown): value is Failure => * refusing party's to give. A refusal the gateway made itself is written by whoever made it, * because only there is it known which field was at fault and which code names the condition. * The protocol's own rendering of a status and a sentence is what is left when neither holds. + * + * Only that third tier is the gateway's to word, and it is therefore the only one that may + * answer with a status other than the failure's: a forwarded body already states its own, and + * a refusal that wrote its own envelope wrote it against the status it refused with. So `mint` + * hands back both, and a protocol cannot state a code in the body that the response contradicts. */ -export const renderFailure = (failure: Failure, protocolEnvelope: () => Record): Record => - renderProtocolError(failure.body, () => failure.envelope ?? protocolEnvelope()); +export const renderFailure = (failure: Failure, mint: (failure: Failure) => RenderedFailure): RenderedFailure => { + if (isJsonObject(failure.body)) return { body: failure.body, status: failure.status }; + if (failure.envelope !== undefined) return { body: failure.envelope, status: failure.status }; + return mint(failure); +}; + +/** The third tier for every protocol whose envelope says nothing about the status — which is + * all of them but Gemini generateContent. */ +export const mintedAs = (envelope: (failure: Failure) => Record) => + (failure: Failure): RenderedFailure => ({ body: envelope(failure), status: failure.status }); + +/** The third tier for the OpenAI-shaped protocols, which is the same envelope in all three. */ +export const mintedErrorEnvelope = mintedAs(failure => renderErrorEnvelope(failure.message)); export interface GatewayFacts { /** What the client sent, before anything read it. Every family hands these over, because From 91289d0d3ad934d4b6cc0bfb0ec422fc30b140ae Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 13:37:32 +0800 Subject: [PATCH 098/156] refactor(chat): let a gateway refusal be its envelope, not bytes to parse back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruling 5: the non-JSON error is a corner case this gateway no longer pays for. `renderFailure` already states the whole of it — a body this protocol can carry is forwarded, anything else is an envelope minted from the status and the message — and what was left was code still shaped for the surface where that was not true. Three refusals the gateway writes itself were serialized so they could be read back. The Anthropic Messages shim built an `ApiErrorResult` around encoded bytes and its one caller decoded and re-parsed them; the server-tool stage stringified its envelope into the message field and set the body to the same object it had just set the envelope to. Both now state the envelope as the envelope: it is what `Failure.envelope` is for, and a refusal this gateway authored has no upstream bytes behind it to round-trip. The test harness had absorbed the difference — it encoded the failure's *message* as the response body, which only read as JSON because the message was a JSON blob. It now renders through `renderFailure`, so what a test decodes is what an edge would have sent. Two comments claimed `Failure.body` survives for a dump reader. It is what the client is answered with, on the tier that forwards it. --- .../chat/openai-responses/server-tools/drive.ts | 11 ++++++++--- .../chat/anthropic-messages/web-search-shim.ts | 13 ++++--------- .../chat/anthropic-messages/web-search-tool.ts | 15 ++++++++------- .../chat/openai-responses/server-tools/stage.ts | 2 +- .../src/data-plane/openai-completions/pipeline.ts | 3 ++- packages/gateway/src/data-plane/pipeline/facts.ts | 6 ++++-- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts index fa3823c657..742039d25d 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/drive.ts @@ -11,6 +11,8 @@ import type { ServerToolRegistration } from '../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'; import { runOpenAIResponsesServerTools } from '../../../../../src/data-plane/chat/openai-responses/server-tools/stage.ts'; import type { ChatGatewayCtx } from '../../../../../src/data-plane/chat/shared/gateway-ctx.ts'; +import type { Failure } from '../../../../../src/data-plane/pipeline/facts.ts'; +import { mintedErrorEnvelope, renderFailure } from '../../../../../src/data-plane/pipeline/facts.ts'; import type { StreamOutcome } from '../../../../../src/data-plane/pipeline/serve.ts'; import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../../../../src/data-plane/shared/telemetry/usage.ts'; import { compose, defer, defineStage, move, run, type Deferred } from '@floway-dev/pipeline'; @@ -45,18 +47,21 @@ const failureOf = (result: Exclude) => ({ message: result.type === 'internal-error' ? result.error.message : new TextDecoder().decode(result.body), }); -/** What the run answered with, in the shape the assertions read. */ +/** What the run answered with, in the shape the assertions read. A refusal is rendered the way + * the edge renders one — through the same three tiers — so a test reads the bytes a client + * would have been sent rather than a restatement of them. */ const resultOf = (facts: Record): Result => { const answer = facts[ANSWER] as { status?: number; message?: string; kind?: string; frames?: unknown }; if (answer.kind === 'stream') { return { type: 'events', events: answer.frames as AsyncIterable>, modelIdentity: undefined } as unknown as Result; } + const rendered = renderFailure(answer as Failure, mintedErrorEnvelope); return { type: 'api-error', source: 'gateway', - status: answer.status!, + status: rendered.status, headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(answer.message!), + body: new TextEncoder().encode(JSON.stringify(rendered.body)), } as Result; }; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts index e90269a1d5..e89df3aa31 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts @@ -839,7 +839,10 @@ export const rewriteAnthropicMessagesWebSearchEventsToNative = async function* ( } }; -const anthropicMessagesWebSearchInvalidRequestBody = (message: string) => ({ +/** The body Anthropic Messages states an unusable tool declaration with. A refusal the gateway + * writes is a refusal in this protocol's own words, which is what makes it readable by the + * client that sent the declaration. */ +export const anthropicMessagesWebSearchInvalidRequestBody = (message: string): Record => ({ type: 'error', error: { type: 'invalid_request_error', @@ -847,14 +850,6 @@ const anthropicMessagesWebSearchInvalidRequestBody = (message: string) => ({ }, }); -export const buildSyntheticInvalidRequestUpstreamError = (message: string) => ({ - type: 'api-error' as const, - source: 'gateway' as const, - status: 400, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(JSON.stringify(anthropicMessagesWebSearchInvalidRequestBody(message))), -}); - /** * The backend a search that will actually run needs. * diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts index 6ef71f68d8..16499c986e 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts @@ -11,7 +11,7 @@ // request rewrite and a response rewrite around one dial. import { - buildSyntheticInvalidRequestUpstreamError, + anthropicMessagesWebSearchInvalidRequestBody, prepareAnthropicMessagesWebSearchInvocation, resolveActiveAnthropicMessagesWebSearchProvider, rewriteAnthropicMessagesWebSearchEventsToNative, @@ -31,12 +31,13 @@ type Answered = M<'response.chat.anthropicMessages' | 'response.usage.billable' /** The gateway's own refusal, in Anthropic's words. It is written here rather than dialled, * because a tool declaration this shim cannot execute would reach the upstream as a body this - * gateway wrote. */ -const refusal = (message: string): Failure => { - const built = buildSyntheticInvalidRequestUpstreamError(message); - const text = new TextDecoder().decode(built.body); - return { status: built.status, message: text, body: JSON.parse(text) as unknown }; -}; + * gateway wrote. The envelope is stated as itself: a refusal this gateway authored has no + * upstream bytes behind it, so there is nothing to serialize and read back. */ +const refusal = (message: string): Failure => ({ + status: 400, + message, + envelope: anthropicMessagesWebSearchInvalidRequestBody(message), +}); /** What the stage is told rather than reaching for: which wire this candidate is reachable on, * which is what decides whether the upstream can carry Anthropic's server tools at all. */ diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/stage.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/stage.ts index 1d68c09909..359efe16c4 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/stage.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/stage.ts @@ -83,7 +83,7 @@ const invalidRequest = ( const envelope = { error: { message, type: errorType, param, code: code === undefined ? 'invalid_request_error' : code }, }; - return { status: 400, message: JSON.stringify(envelope), body: envelope, envelope }; + return { status: 400, message, envelope }; }; /** diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index d854ed854e..ae38f41204 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -311,7 +311,8 @@ const refusal = async (response: Response): Promise => { return { status: response.status, message: text, body: JSON.parse(text) as unknown }; } catch { // A refusal that is not JSON is still a refusal, and its text is what the client is - // told; there is simply no parsed body for a dump reader to open. + // told — as this protocol's own envelope, minted from the status and these words, because + // a body this protocol cannot carry is not one to hand on. return { status: response.status, message: text }; } }; diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index 2b408e3a3a..f5b1cd41a1 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -42,8 +42,10 @@ export interface BillableEntity { export interface Failure { readonly status: number; readonly message: string; - /** The upstream's own body, when there was one. A client is not owed the upstream's - * exact bytes, but a dump reader is owed what actually came back. */ + /** The upstream's own body, parsed, when it sent one this protocol can carry. It is what the + * client is answered with — the refusing party's words rather than a restatement of them — + * and a body that was not JSON at all leaves this absent, because there is nothing to carry + * and an envelope minted from the status and the message is what stands in for it. */ readonly body?: unknown; /** What the gateway itself would write, when the refusal is its own. A protocol's envelope * carries more than a status and a sentence — which field was at fault, which code names From 3059299f218d96f828e1e0ee377119851e3d3d5d Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 14:23:45 +0800 Subject: [PATCH 099/156] chore(migrations): make room for the flag-id rewrite The naming branch takes `0084` for the flag-id rewrite, so this one moves to `0085`. Nothing references it by name and neither has been applied anywhere, so the number is the whole of the change; the two touch different tables and would have run correctly in either order regardless. --- ...drop_edge_dump_records.sql => 0085_drop_edge_dump_records.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/gateway/migrations/{0084_drop_edge_dump_records.sql => 0085_drop_edge_dump_records.sql} (100%) diff --git a/packages/gateway/migrations/0084_drop_edge_dump_records.sql b/packages/gateway/migrations/0085_drop_edge_dump_records.sql similarity index 100% rename from packages/gateway/migrations/0084_drop_edge_dump_records.sql rename to packages/gateway/migrations/0085_drop_edge_dump_records.sql From 76baf2f6ef8635d34d87b6f05e7f6ae2876b5b94 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 15:20:57 +0800 Subject: [PATCH 100/156] refactor: say why, not what the code used to be MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ten comments justified a decision by naming what the surface this replaced did. That reads as a citation and resolves to nothing once this ships — the reader has no replaced surface to look at. Each now states the property itself, and where the point really is a contrast, contrasts with the alternative a reader can still evaluate: what a target-API guard inside each rule would have said, what forwarding an unreadable body under a 200 would have claimed. No behavior and no assertions change; every one of these is a comment. --- .../data-plane/chat/translated-wires_test.ts | 13 ++++++------- .../chat/translation-refusal_test.ts | 18 +++++++++--------- .../openai-completions-pipeline_test.ts | 6 +++--- .../openai-embeddings/failures_test.ts | 7 ++++--- .../__tests__/data-plane/rerank-status_test.ts | 3 ++- .../__tests__/data-plane/rerank/serve_test.ts | 6 +++--- .../chat/gemini-generate-content/pipeline.ts | 7 ++++--- .../src/data-plane/openai-audio/pipeline.ts | 2 +- .../src/data-plane/pipeline/upstream-body.ts | 4 ++-- 9 files changed, 34 insertions(+), 32 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 92c6d413a0..6ac8ce879b 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -9,8 +9,8 @@ // Beside the matrix, three things that are about the fork rather than about a pair: that // failover moves from a native candidate onto a translated one, that a pair's own refusal // rewrite survives the trip, and that a rule which speaks about one protocol's wire does not -// run on a turn that leaves for another — which is what the replaced surface said with -// `ctx.targetApi !== ` and what position says here. +// run on a turn that leaves for another — which a target-API guard inside each rule could also +// say, and which position says here without a rule having to know where it sits. import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -414,8 +414,8 @@ describe('the fork over wires', () => { const rendered = facts['response.chat.anthropicMessages.rendered'] as { type: string; error: { type: string; message: string } }; expect(rendered.type).toBe('error'); expect(rendered.error.type).toBe('permission_error'); - // The upstream's own words survive as the sentence, which is what the replaced surface did - // with a body it could not forward. + // The upstream's own words survive as the sentence, which is all that can be kept of a + // body written in a protocol this client does not read. expect(rendered.error.message).toContain('no'); await drain(); }); @@ -511,9 +511,8 @@ describe('a rule that speaks about one protocol-s wire', () => { // And a rule the *other* wire owns stays off this turn. `stream_options` is an OpenAI Chat // Completions field, so an OpenAI Chat Completions turn dialled over Anthropic Messages must - // not carry it — - // which is what the replaced surface said with a target-API guard and - // what position says here. + // not carry it — which a target-API guard inside the rule could also say, and which position + // says here. it('does not carry another wire-s request rules onto the wire it dialled', async () => { let sent: Record | undefined; resolves([candidate('up_a', { anthropicMessages: {} }, { diff --git a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts index 76c885f710..136bad71d4 100644 --- a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts @@ -1,19 +1,19 @@ // A body the target protocol cannot represent is the client's fault. // // A translated wire runs the client's body through a translator, and a translator refuses input -// it would otherwise have to coerce or silently drop. The replaced surface rendered that refusal -// as a 400 in the client's own protocol; a run that let it escape as a throw would answer 500 with -// a stack trace, telling the caller that the gateway broke on a request that was simply not -// serveable over that wire. +// it would otherwise have to coerce or silently drop. That refusal is rendered as a 400 in the +// client's own protocol; a run that let it escape as a throw would answer 500 with a stack +// trace, telling the caller that the gateway broke on a request that was simply not serveable +// over that wire. // -// It is answered rather than thrown for a second reason the replaced surface had no way to -// express: the verdict belongs to one candidate. The next candidate may be reachable on a wire -// whose protocol *can* carry the body, and failover re-runs the suffix to find out. +// It is answered rather than thrown for a second reason: the verdict belongs to one candidate. +// The next candidate may be reachable on a wire whose protocol *can* carry the body, and +// failover re-runs the suffix to find out. // // Both of Gemini generateContent's actions cross a protocol boundary and both are covered: generation reaches // its target through a handoff, and `:countTokens` through the pair that asks the question in -// Anthropic Messages. The replaced surface answered a refusal the same way on either, because what the -// caller reads is the same protocol whichever action it asked for. +// Anthropic Messages. A refusal is answered the same way on either, because what the caller +// reads is the same protocol whichever action it asked for. import { test } from 'vitest'; diff --git a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts index db633e25ed..2bdc7e6a9a 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts @@ -285,9 +285,9 @@ describe('the OpenAI Completions pipeline', () => { expect(outcome.failed).toBe(false); }); - // A run that reached no upstream bills nothing and samples nothing, which is what the - // replaced surface did: `recordPerformance` returns early without an attempt's telemetry, - // and there is no attempt. Settlement still runs — it is unconditional — and finds an + // A run that reached no upstream bills nothing and samples nothing: + // `recordPerformance` returns early without an attempt's telemetry, and there is no + // attempt. Settlement still runs — it is unconditional — and finds an // empty billed set, which is how "we did not call an upstream" is said. it('bills nothing and samples nothing when no upstream was reached', async () => { vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [], sawModel: false, failedUpstreams: [] }); diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts index 3d812f671f..b06b87e761 100644 --- a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts @@ -76,9 +76,10 @@ test('a usage write that fails leaves the answer alone and still reports', async } }); -// The replaced surface forwarded such a body verbatim with the upstream's 200. A protocol -// that requires JSON and did not get JSON has no answer to serve, so the gateway says so -// itself — it cannot claim to have served a request whose answer it never read. +// A protocol that requires JSON and did not get JSON has no answer to serve, so the gateway +// says so itself — it cannot claim to have served a request whose answer it never read. +// Forwarding the body verbatim under the upstream's 200 would report success for a turn +// nothing in this gateway could read. test('a 2xx body the OpenAI Embeddings protocol cannot read is refused rather than forwarded', async () => { const { apiKey, repo } = await setupAppTest(); await registerOpenAIEmbeddingsUpstream(repo); diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts index e7507b3c15..0294b84032 100644 --- a/packages/gateway/__tests__/data-plane/rerank-status_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -1,7 +1,8 @@ // The two families that read their answer to the end, driven end to end. Until the review, // neither could express a status at all: an upstream 429, a resolver's 404 and a 400 all // reached the client as a 200 carrying an error envelope, which is not a difference the -// no-passthrough ruling asks for — the replaced surface forwarded the upstream's status. +// no-passthrough ruling asks for — declining to forward a body is not declining to forward +// a status. import { beforeEach, describe, expect, it, vi } from 'vitest'; diff --git a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts index ea74bed97a..b1bd422dbc 100644 --- a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts @@ -516,9 +516,9 @@ test('cross-protocol success still validates result items before rendering', asy assertEquals(performance.errorsNoOutput, 1); }); -// The replaced surface forwarded these bytes verbatim with the upstream's 200. Rerank is a -// JSON protocol, so a body that is not JSON at all is no answer to serve and the gateway says -// so itself rather than passing on something it never read. +// Rerank is a JSON protocol, so a body that is not JSON at all is no answer to serve and the +// gateway says so itself rather than passing on something it never read. Forwarding those +// bytes verbatim under the upstream's 200 would report success for a turn nothing could read. test('same-protocol answer that is not JSON is refused rather than forwarded', async () => { const { apiKey, repo } = await setupAppTest(); await saveRerankUpstream(repo, { protocol: 'jina-v1' }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index 310ce6888d..6ba320d1bf 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -30,9 +30,10 @@ // than another wire under this pipeline — no stream, no billable turn, and a // `{ totalTokens }` envelope of its own — so it is a chain of its own in `count-tokens.ts`. // -// What is not built, stated rather than implied: the Google-RPC envelope the replaced surface -// wrote into a client's own stream for a turn that ended without a terminal event. The chain -// detects one and leaves the run as a throw rather than as that 502 envelope. +// A turn that ends without a terminal event is left as a throw rather than answered with a +// Google-RPC envelope written into the client's own stream. Half a stream followed by an error +// object is a turn that reads as complete-then-refused, and the client has already been handed +// content it will keep; a run that ends is the honest report of an upstream that stopped. // // A refusal is the one place where having no wire of its own changes what a client is owed. // Every other family can hand an upstream's own error object on, because the upstream spoke diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index 8ee16aa8e9..24863d1bbe 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -57,7 +57,7 @@ export interface OpenAIAudioTranscriptionStreamOutcome { readonly billable: readonly BillableEntity[]; /** An upstream that stopped before `transcript.text.done` answered 200 and then did not * finish what it started, which is a failed request however much of the transcript - * reached the client — the same reading the replaced surface made. */ + * reached the client. */ readonly failed: boolean; } diff --git a/packages/gateway/src/data-plane/pipeline/upstream-body.ts b/packages/gateway/src/data-plane/pipeline/upstream-body.ts index 068b4cce65..84f5484345 100644 --- a/packages/gateway/src/data-plane/pipeline/upstream-body.ts +++ b/packages/gateway/src/data-plane/pipeline/upstream-body.ts @@ -43,8 +43,8 @@ export const unreadableBody = (response: Response, body: UpstreamBody, protocolN * Nothing in the domain throws. A connection that was refused, timed out or was reset is an * outcome failover has to be able to see — a run that ends there has more candidates to try — * so the ending catches whatever the platform raised and hands it up like any other failure. - * The status is the one the replaced surface answered with: the gateway reached no upstream, - * which is what 502 says. + * The status says what happened rather than what the error was: the gateway reached no + * upstream at all, which is what 502 states. */ export const dialFailure = (error: unknown): Failure => ({ status: 502, From 3df0868159d737ca508c753a4ed6a0394aaeb621 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 15:22:06 +0800 Subject: [PATCH 101/156] test(web): wait for the restored trace rather than drain once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `source-mapped-stack-status_test.tsx` failed twice in a row under `pnpm run verify` and passed every time the suite ran alone — the signature of a test that races rather than one that is wrong. `settle()` is one macrotask hop, and its contract says that is enough because everything queued before it is microtasks. Restoring a frame breaks that premise: it fetches the script, reads the body, follows `sourceMappingURL`, fetches the map and reads that body too. Two `Response` body reads are real stream reads, so how many turns the chain takes is a property of the machine, and under a loaded `verify` it takes more than one. The three assertions that wait for a restoration to finish now use `waitFor`, so they wait for the outcome instead of assuming a turn count. The fourth keeps `settle()` deliberately: it asserts that a superseded restoration writes *nothing*, and draining exactly what the release queued is the window in which it could have written. Nothing here is new to this branch — the test and the hook are unchanged from `main`. What is new is 170-odd more test files running beside it. --- .../lib/source-mapped-stack-status_test.tsx | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/apps/web/__tests__/lib/source-mapped-stack-status_test.tsx b/apps/web/__tests__/lib/source-mapped-stack-status_test.tsx index 6285974354..f2d444d0bd 100644 --- a/apps/web/__tests__/lib/source-mapped-stack-status_test.tsx +++ b/apps/web/__tests__/lib/source-mapped-stack-status_test.tsx @@ -1,4 +1,4 @@ -import { act, screen } from '@testing-library/react'; +import { act, screen, waitFor } from '@testing-library/react'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { useSourceMappedStack } from '../../src/lib/source-mapped-stack'; @@ -60,8 +60,10 @@ describe('what the error page is told about its trace', () => { it('replaces the trace once the maps land', async () => { vi.stubEnv('DEV', false); renderInApp(); - await settle(); - expect(read()).toEqual({ status: 'settled', stack: 'Error: boom\n at handler (/src/first.ts:1:1)' }); + // Waited for rather than drained once: restoring a frame fetches the script and then its + // map, and reading each body is a real stream read rather than a microtask, so how many + // turns the chain takes is the machine's business and not this test's. + await waitFor(() => expect(read()).toEqual({ status: 'settled', stack: 'Error: boom\n at handler (/src/first.ts:1:1)' })); }); it('keeps the minified trace and says so when the maps cannot be read', async () => { @@ -69,8 +71,7 @@ describe('what the error page is told about its trace', () => { vi.stubGlobal('fetch', vi.fn(() => Promise.resolve(new Response('', { status: 503 })))); vi.spyOn(console, 'error').mockImplementation(() => {}); renderInApp(); - await settle(); - expect(read()).toEqual({ status: 'failed', stack: RAW }); + await waitFor(() => expect(read()).toEqual({ status: 'failed', stack: RAW })); }); it('goes back to waiting when a second failure replaces the first', async () => { @@ -102,9 +103,11 @@ describe('what the error page is told about its trace', () => { const { rerender } = renderInApp(); const second = `Error: later\n at other (${SCRIPT}:1:1)`; rerender(); - await settle(); - expect(read()).toEqual({ status: 'settled', stack: 'Error: later\n at other (/src/first.ts:1:1)' }); + await waitFor(() => expect(read()).toEqual({ status: 'settled', stack: 'Error: later\n at other (/src/first.ts:1:1)' })); + // The superseded restoration finishes here, and what it must not do is write. A settle is + // the right instrument for that one: the assertion is that nothing changed, so draining + // everything the release queued is exactly the window it could have changed something in. await act(async () => { releaseFirst(new Response('const a=1;\n//# sourceMappingURL=chunk.js.map')); }); From fbbd66788950a6031a9a853950f626214f34c351 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 16:26:39 +0800 Subject: [PATCH 102/156] refactor(gemini-generate-content): rewrite the affinity egress without writing into a frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last clone in the pipeline, and the reason it was there. The signature transducer deep-cloned every event on the way in so the rules could write into the copy: `delete part.thoughtSignature`, `parts[i].thoughtSignature = …`, `candidate.content.parts = …`, `next.candidates = …`. Ruling 2 admits no carve-out for a frame — nothing in the pipeline is written into, and nothing in it needs a clone — so the pass is a rebuild and the clone is gone. Two things kept the old shape honest and both were keyed on object identity, which a rebuild destroys. A `WeakSet` marked events the pass had suppressed, and another marked candidates a relocation had emptied; both are now carried. The held event travels as `{ event, suppressed }`, and each relocation hands on a `CandidatePair` stating what the pair now is and which side it emptied. The five `void` mutators — `normalizeElementSignatures`, the three `relocate*` and `removeRelocatedSignatureParts` — return values instead. Nothing about the algorithm changes: same sliding window, same relocation rules, same anchoring, same vendor references. The 28 existing tests pass unchanged, which is the point — this is a rewrite of how the pass expresses itself, not of what it decides. The new test is the property the rewrite exists for: a deeply frozen source survives the pass. Verified by mutation — making `withParts` assign into the candidate it was handed fails it with "Cannot assign to read only property 'parts'", where the old code would have silently written into the caller's event. The other clones the sweep turned up went the same way. The stored-items scratchpad copied private payloads in and out, though the payloads it is handed come out of the record already frozen and its one reader splices a sub-object of them into an array. `cloneAnthropicMessagesUsageIterations` deep-copied an opaque vendor array at three hops; nothing reads inside an entry and nothing writes one, and the two tests that pinned the copy asserted it by mutating the source themselves — they now assert what the snapshot actually owes the array. What stays is stated where it stays. `getItemById` copies because what hydration builds from that row is handed to `move()`, and handing over a live cache entry would deep-freeze the cache. --- .../affinity/egress_test.ts | 46 ++ .../affinity/egress.ts | 469 +++++++++++------- .../chat/openai-responses/items/store.ts | 17 +- .../anthropic-messages/usage_test.ts | 47 +- .../src/anthropic-messages/reassemble.ts | 5 +- .../protocols/src/anthropic-messages/usage.ts | 16 +- 6 files changed, 366 insertions(+), 234 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/affinity/egress_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/affinity/egress_test.ts index 561fb5a88a..7dd6de8413 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/affinity/egress_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/affinity/egress_test.ts @@ -530,4 +530,50 @@ describe('Gemini generateContent affinity egress', () => { }); await expect(iterator.next()).rejects.toThrow('upstream failed'); }); + + // The property the whole shape of this transducer rests on: a frame it is handed belongs to + // the layer that produced it. The events used to be deep-cloned on the way in precisely + // because the rules then wrote into them; nothing writes now, so a frozen source has to + // survive the pass unchanged — and the relocations are exercised here rather than asserted + // about, because a rewrite that reached for `delete` or an index assignment throws in strict + // mode instead of quietly corrupting the caller's event. + test('rewrites a deeply frozen source without writing into it', async () => { + const freeze = (value: T): T => { + if (value === null || typeof value !== 'object') return value; + for (const entry of Object.values(value as Record)) freeze(entry); + return Object.freeze(value); + }; + const frozen = (event: GeminiGenerateContentStreamEvent): ProtocolFrame => eventFrame(freeze(event)); + const source: ProtocolFrame[] = [ + // A signature-only leader, relocated forward onto the next event's content. + frozen({ + candidates: [{ index: 0, content: { role: 'model', parts: [{ thoughtSignature: 'lead' }] } }], + }), + // Content that continues into the third event, whose signature is pulled back onto it. + frozen({ + candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'hello' }] } }], + usageMetadata: { totalTokenCount: 1 }, + }), + frozen({ + candidates: [{ + index: 0, + content: { role: 'model', parts: [{ text: ' world', thoughtSignature: 'tail' }] }, + finishReason: 'STOP', + }], + }), + ]; + const before = JSON.stringify(source); + + const output: ProtocolFrame[] = []; + for await (const frame of wrapGeminiGenerateContentAffinityEgress(frames(source), { codec: immediateCodec, affinity })) { + output.push(frame); + } + + // The source is what it was, and the pass still did its work. + expect(JSON.stringify(source)).toBe(before); + const signatures = output.flatMap(frame => + (frame.type === 'event' && 'candidates' in frame.event ? frame.event.candidates ?? [] : []) + .flatMap(candidate => candidate.content.parts.flatMap(part => part.thoughtSignature ?? []))); + expect(signatures.some(signature => signature.startsWith('wrapped:'))).toBe(true); + }); }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/egress.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/egress.ts index 919b1eee5c..2275a66b2d 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/egress.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/egress.ts @@ -1,5 +1,5 @@ import type { AffinityEgressOptions } from '../../shared/affinity/index.ts'; -import { captureExtras, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { captureExtras, eventFrame, withIndexesChanged, withKeysChanged, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { GeminiGenerateContentCandidate, GeminiGenerateContentPart, GeminiGenerateContentResult, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; import { GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS, GEMINI_GENERATE_CONTENT_RESULT_KEYS } from '@floway-dev/protocols/gemini-generate-content'; @@ -30,8 +30,12 @@ export const wrapGeminiGenerateContentAffinityEgress = async function* ( options: AffinityEgressOptions, ): AsyncGenerator> { const anchoredCandidates = new Set(); - const suppressedEvents = new WeakSet(); - let pending: GeminiGenerateContentResult | undefined; + // The one event held back, and what the pass that produced it decided about emitting it. + // Both used to be recorded against the event object — the verdict in a `WeakSet`, the + // rewrites by assignment — which only works while an event can be written in place. Nothing + // here is written in place any more, so a pass hands back new events and the verdict travels + // beside the one it is about rather than being looked up by identity. + let pending: PendingEvent | undefined; const iterator = frames[Symbol.asyncIterator](); let sourceCompleted = false; @@ -42,15 +46,8 @@ export const wrapGeminiGenerateContentAffinityEgress = async function* ( result = await iterator.next(); } catch (error) { if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - undefined, - anchoredCandidates, - suppressedEvents, - options, - false, - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + const flushed = await flushPending(pending, anchoredCandidates, options, false); + if (flushed !== undefined) yield eventFrame(flushed); pending = undefined; } throw error; @@ -62,15 +59,8 @@ export const wrapGeminiGenerateContentAffinityEgress = async function* ( const frame = result.value; if (frame.type !== 'event') { if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - undefined, - anchoredCandidates, - suppressedEvents, - options, - frame.type === 'done', - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + const flushed = await flushPending(pending, anchoredCandidates, options, frame.type === 'done'); + if (flushed !== undefined) yield eventFrame(flushed); pending = undefined; } yield frame; @@ -78,72 +68,118 @@ export const wrapGeminiGenerateContentAffinityEgress = async function* ( } if ('error' in frame.event) { if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - undefined, - anchoredCandidates, - suppressedEvents, - options, - false, - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + const flushed = await flushPending(pending, anchoredCandidates, options, false); + if (flushed !== undefined) yield eventFrame(flushed); } yield frame; return; } - const next = cloneGeminiGenerateContentEvent(frame.event); - if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - next, - anchoredCandidates, - suppressedEvents, - options, - false, - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + if (pending === undefined) { + pending = { event: frame.event, suppressed: false }; + continue; } - pending = next; + const pass = await passOverEventPair(pending, frame.event, anchoredCandidates, options, false); + if (!pass.current.suppressed) yield eventFrame(pass.current.event); + pending = pass.next; } } finally { if (!sourceCompleted) await iterator.return?.(); } if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - undefined, - anchoredCandidates, - suppressedEvents, - options, - false, - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + const flushed = await flushPending(pending, anchoredCandidates, options, false); + if (flushed !== undefined) yield eventFrame(flushed); } }; -const cloneGeminiGenerateContentEvent = (event: GeminiGenerateContentResult): GeminiGenerateContentResult => structuredClone(event); +/** One event, and whether the pass that produced it already decided it is not to be emitted. */ +interface PendingEvent { + readonly event: GeminiGenerateContentResult; + readonly suppressed: boolean; +} + +/** The last pass an event gets: there is no successor, so nothing can move forward out of it. + * Returns what to emit, or nothing where the event was suppressed. */ +const flushPending = async ( + pending: PendingEvent, + anchoredCandidates: Set, + options: AffinityEgressOptions, + successfulTerminalBoundary: boolean, +): Promise => { + const pass = await passOverEventPair(pending, undefined, anchoredCandidates, options, successfulTerminalBoundary); + return pass.current.suppressed ? undefined : pass.current.event; +}; + +/** What one relocation hands on: the pair as it now stands, and whether either side was left + * with no Parts at all — which is the pair's own statement that the candidate should not be + * emitted. This used to be a `WeakSet` of the candidates a relocation emptied; a rebuilt + * candidate is a different object, so the verdict is carried rather than looked up. */ +interface CandidatePair { + readonly current: GeminiGenerateContentCandidate; + readonly next: GeminiGenerateContentCandidate | undefined; + readonly currentRemoved: boolean; + readonly nextRemoved: boolean; +} + +/** A candidate with different Parts, or the same candidate where they did not change. */ +const withParts = ( + candidate: GeminiGenerateContentCandidate, + parts: readonly GeminiGenerateContentPart[], +): GeminiGenerateContentCandidate => + parts === candidate.content.parts + ? candidate + : { ...candidate, content: { ...candidate.content, parts: parts as GeminiGenerateContentPart[] } }; + +/** An event with different candidates. An event that declared none keeps declaring none: the + * absent key and the empty array say different things to a client. */ +const withCandidates = ( + event: GeminiGenerateContentResult, + candidates: readonly GeminiGenerateContentCandidate[], +): GeminiGenerateContentResult => + event.candidates === undefined || candidates === event.candidates + ? event + : { ...event, candidates: candidates as GeminiGenerateContentCandidate[] }; -const wrapGeminiGenerateContentEventAffinity = async ( - current: GeminiGenerateContentResult, - next: GeminiGenerateContentResult | undefined, +/** + * One pass of the sliding window: the held event, the one behind it, and what becomes of both. + * + * Every rewrite here is a rebuild. A frame this transducer was handed belongs to the layer that + * produced it, and the events it hands on are new objects that share everything the pass did not + * touch — so a Part no rule spoke about is the same Part, and the pass costs what it changed + * rather than a copy of the event. + */ +const passOverEventPair = async ( + pending: PendingEvent, + nextEvent: GeminiGenerateContentResult | undefined, anchoredCandidates: Set, - suppressedEvents: WeakSet, options: AffinityEgressOptions, successfulTerminalBoundary: boolean, -): Promise => { +): Promise<{ readonly current: PendingEvent; readonly next: PendingEvent | undefined }> => { + const current = pending.event; const currentHadCandidates = (current.candidates?.length ?? 0) > 0; - const nextHadCandidates = (next?.candidates?.length ?? 0) > 0; - const removedCandidates = new WeakSet(); - - for (const candidate of current.candidates ?? []) { - const nextCandidate = next?.candidates?.find(nextCandidate => nextCandidate.index === candidate.index); - normalizeElementSignatures(candidate); - if (nextCandidate !== undefined) normalizeElementSignatures(nextCandidate); - relocateSignatureOnlyForward(candidate, nextCandidate, removedCandidates); - relocateSignatureOnlyBackward(candidate, nextCandidate, removedCandidates); - relocateContinuationSignature(candidate, nextCandidate); + const nextHadCandidates = (nextEvent?.candidates?.length ?? 0) > 0; + + // The two arrays under construction. Their elements are replaced, never written into: what + // goes back is whatever the rules built, and an untouched candidate goes back as itself. + const builtCurrent = [...(current.candidates ?? [])]; + const builtNext = nextEvent === undefined ? undefined : [...(nextEvent.candidates ?? [])]; + const removedCurrent = new Set(); + const removedNext = new Set(); + + for (let position = 0; position < builtCurrent.length; position += 1) { + const nextPosition = builtNext?.findIndex(entry => entry.index === builtCurrent[position].index) ?? -1; + const relocated = relocateContinuationSignature(relocateSignatureOnlyBackward(relocateSignatureOnlyForward({ + current: normalizeElementSignatures(builtCurrent[position]), + next: nextPosition >= 0 ? normalizeElementSignatures(builtNext![nextPosition]) : undefined, + currentRemoved: false, + nextRemoved: false, + }))); + let candidate = relocated.current; + const nextCandidate = relocated.next; + if (relocated.currentRemoved) removedCurrent.add(position); + if (relocated.nextRemoved && nextPosition >= 0) removedNext.add(nextPosition); + const firstIndexes = firstElementIndexes(candidate.content.parts); const firstHasNatural = firstIndexes.some(index => candidate.content.parts[index].thoughtSignature !== undefined); const signatureOnlyNatural = firstIndexes.length === 0 @@ -159,150 +195,192 @@ const wrapGeminiGenerateContentEventAffinity = async ( && candidate.content.parts.slice(lastFirstContentIndex + 1).some(hasPartContent); const firstElementClosed = startsAnotherElementInCurrentEvent || candidate.finishReason !== undefined - || (next !== undefined && nextCandidate === undefined) - || (nextCandidate !== undefined && nextContent === undefined && !removedCandidates.has(nextCandidate)) + || (nextEvent !== undefined && nextCandidate === undefined) + || (nextCandidate !== undefined && nextContent === undefined && !relocated.nextRemoved) || (nextCandidate !== undefined && nextContent !== undefined && !continuesInNextEvent) - || (next === undefined && successfulTerminalBoundary); + || (nextEvent === undefined && successfulTerminalBoundary); + const wrapped: GeminiGenerateContentPart[] = []; for (const part of candidate.content.parts) { - if (part.thoughtSignature === undefined) continue; - part.thoughtSignature = await options.codec.wrap(part.thoughtSignature, options.affinity, 'gemini-generate-content.part.thoughtSignature'); + wrapped.push(part.thoughtSignature === undefined ? part : { + ...part, + thoughtSignature: await options.codec.wrap(part.thoughtSignature, options.affinity, 'gemini-generate-content.part.thoughtSignature'), + }); } + candidate = withParts(candidate, wrapped); - if (anchoredCandidates.has(candidate.index)) continue; - if (firstHasNatural || signatureOnlyNatural) { - anchoredCandidates.add(candidate.index); - } else if (firstContentIndex !== undefined && firstElementClosed) { - candidate.content.parts[firstContentIndex] = { - ...candidate.content.parts[firstContentIndex], - thoughtSignature: await options.codec.wrap(undefined, options.affinity, 'gemini-generate-content.part.thoughtSignature'), - }; - anchoredCandidates.add(candidate.index); - } else if (candidate.finishReason !== undefined) { - candidate.content.parts.push({ - thoughtSignature: await options.codec.wrap( - undefined, - options.affinity, - 'gemini-generate-content.part.thoughtSignature', - ), - }); - anchoredCandidates.add(candidate.index); + if (!anchoredCandidates.has(candidate.index)) { + if (firstHasNatural || signatureOnlyNatural) { + anchoredCandidates.add(candidate.index); + } else if (firstContentIndex !== undefined && firstElementClosed) { + candidate = withParts(candidate, withIndexesChanged(candidate.content.parts, new Map([[firstContentIndex, { + ...candidate.content.parts[firstContentIndex], + thoughtSignature: await options.codec.wrap(undefined, options.affinity, 'gemini-generate-content.part.thoughtSignature'), + }]]))); + anchoredCandidates.add(candidate.index); + } else if (candidate.finishReason !== undefined) { + candidate = withParts(candidate, [...candidate.content.parts, { + thoughtSignature: await options.codec.wrap(undefined, options.affinity, 'gemini-generate-content.part.thoughtSignature'), + }]); + anchoredCandidates.add(candidate.index); + } } - } - if (next?.candidates !== undefined) { - next.candidates = next.candidates.filter(candidate => !removedCandidates.has(candidate)); - } - if (current.candidates !== undefined) { - current.candidates = current.candidates.filter(candidate => !removedCandidates.has(candidate)); + builtCurrent[position] = candidate; + if (nextPosition >= 0 && nextCandidate !== undefined) builtNext![nextPosition] = nextCandidate; } - const currentEmptied = currentHadCandidates && current.candidates?.length === 0; - const nextEmptied = nextHadCandidates && next?.candidates?.length === 0; - if (currentEmptied && next !== undefined && !nextEmptied) { - mergeEventMetadata(current, next, next); - clearEventMetadata(current); - suppressedEvents.add(current); - } else if (nextEmptied && next !== undefined && !currentEmptied) { - mergeEventMetadata(current, next, current); - clearEventMetadata(next); - suppressedEvents.add(next); + + const keptCurrent = builtCurrent.filter((_candidate, position) => !removedCurrent.has(position)); + const keptNext = builtNext?.filter((_candidate, position) => !removedNext.has(position)); + const currentEmptied = currentHadCandidates && keptCurrent.length === 0; + const nextEmptied = nextHadCandidates && keptNext?.length === 0; + + let currentOut = withCandidates(current, keptCurrent); + let nextOut = nextEvent === undefined ? undefined : withCandidates(nextEvent, keptNext ?? []); + let currentSuppressed = pending.suppressed; + let nextSuppressed = false; + + // An event whose every candidate moved into its neighbour has nothing left to say on its own, + // so what it was carrying beside them moves too and the event itself is not emitted. + if (currentEmptied && nextOut !== undefined && !nextEmptied) { + nextOut = mergedEventMetadata(currentOut, nextOut, nextOut); + currentOut = withoutEventMetadata(currentOut); + currentSuppressed = true; + } else if (nextEmptied && nextOut !== undefined && !currentEmptied) { + currentOut = mergedEventMetadata(currentOut, nextOut, currentOut); + nextOut = withoutEventMetadata(nextOut); + nextSuppressed = true; } - return current; + + return { + current: { event: currentOut, suppressed: currentSuppressed }, + next: nextOut === undefined ? undefined : { event: nextOut, suppressed: nextSuppressed }, + }; }; -const normalizeElementSignatures = (candidate: GeminiGenerateContentCandidate): void => { +/** Every signature a logical element carries, collapsed onto that element's first + * content-bearing Part, and the signature-only Parts that leaves behind dropped. */ +const normalizeElementSignatures = (candidate: GeminiGenerateContentCandidate): GeminiGenerateContentCandidate => { const parts = candidate.content.parts; - const signatureParts = new Set(parts.filter(part => part.thoughtSignature !== undefined)); - for (const indexes of logicalElementGroups(parts)) normalizeElementSignature(parts, indexes); - removeRelocatedSignatureParts(candidate, signatureParts); + const carriedSignature = new Set(parts.flatMap((part, index) => part.thoughtSignature === undefined ? [] : [index])); + // The groups are index lists read off the Parts as they arrived, and collapsing a signature + // never changes how many Parts there are, so they stay valid across the fold. + let folded: readonly GeminiGenerateContentPart[] = parts; + for (const indexes of logicalElementGroups(parts)) folded = normalizeElementSignature(folded, indexes); + return withParts(candidate, keptAfterRelocation(folded, carriedSignature)); }; -const normalizeElementSignature = (parts: GeminiGenerateContentPart[], indexes: readonly number[]): void => { +const normalizeElementSignature = ( + parts: readonly GeminiGenerateContentPart[], + indexes: readonly number[], +): readonly GeminiGenerateContentPart[] => { const signatures = indexes.flatMap(index => { const signature = parts[index].thoughtSignature; return signature === undefined ? [] : [signature]; }); - if (signatures.length === 0) return; + if (signatures.length === 0) return parts; const targetIndex = indexes.find(index => hasPartContent(parts[index])) ?? indexes.at(-1); if (targetIndex === undefined) throw new Error('Gemini generateContent signature group has no target Part'); - for (const index of indexes) delete parts[index].thoughtSignature; - parts[targetIndex].thoughtSignature = signatures[signatures.length - 1]; + const rewrites = new Map( + indexes.map(index => [index, withKeysChanged(parts[index], { thoughtSignature: undefined })]), + ); + rewrites.set(targetIndex, withKeysChanged(parts[targetIndex], { thoughtSignature: signatures[signatures.length - 1] })); + return parts.map((part, index) => rewrites.get(index) ?? part); }; -const relocateSignatureOnlyForward = ( - current: GeminiGenerateContentCandidate, - next: GeminiGenerateContentCandidate | undefined, - removedCandidates: WeakSet, -): void => { +const relocateSignatureOnlyForward = (pair: CandidatePair): CandidatePair => { + const { current, next } = pair; if ( next === undefined || current.finishReason !== undefined || current.content.parts.some(hasPartContent) - ) return; - const relocated = new Set(current.content.parts.filter(part => part.thoughtSignature !== undefined)); - const signature = relocated.values().next().value?.thoughtSignature; + ) return pair; + const relocated = new Set(current.content.parts.flatMap((part, index) => part.thoughtSignature === undefined ? [] : [index])); + const signature = current.content.parts.find(part => part.thoughtSignature !== undefined)?.thoughtSignature; const targetIndex = firstElementIndexes(next.content.parts).find(index => hasPartContent(next.content.parts[index])); - if (signature === undefined || targetIndex === undefined) return; - next.content.parts[targetIndex].thoughtSignature ??= signature; - for (const part of relocated) delete part.thoughtSignature; - removeRelocatedSignatureParts(current, relocated, removedCandidates); - if (current.content.parts.length === 0) transferCandidateMetadataForward(current, next); + if (signature === undefined || targetIndex === undefined) return pair; + + // A target that already carries one keeps it: this moves a signature into a gap, never over + // an upstream's own. + const carried = next.content.parts[targetIndex].thoughtSignature !== undefined ? next : withParts( + next, + withIndexesChanged(next.content.parts, new Map([[targetIndex, withKeysChanged(next.content.parts[targetIndex], { thoughtSignature: signature })]])), + ); + const strippedParts = keptAfterRelocation( + current.content.parts.map(part => part.thoughtSignature === undefined ? part : withKeysChanged(part, { thoughtSignature: undefined })), + relocated, + ); + const stripped = withParts(current, strippedParts); + return strippedParts.length > 0 + ? { ...pair, current: stripped, next: carried } + : { ...pair, current: stripped, next: withForwardedCandidateMetadata(stripped, carried), currentRemoved: true }; }; -const relocateSignatureOnlyBackward = ( - current: GeminiGenerateContentCandidate, - next: GeminiGenerateContentCandidate | undefined, - removedCandidates: WeakSet, -): void => { +const relocateSignatureOnlyBackward = (pair: CandidatePair): CandidatePair => { + const { current, next } = pair; if ( next === undefined || current.finishReason !== undefined || next.content.parts.some(hasPartContent) - ) return; + ) return pair; const targetIndex = firstContentIndexOfLastElement(current.content.parts); - if (targetIndex === undefined) return; - const signatureOnly = next.content.parts.find(part => part.thoughtSignature !== undefined); - if (signatureOnly?.thoughtSignature === undefined) return; + if (targetIndex === undefined) return pair; + const signatureIndex = next.content.parts.findIndex(part => part.thoughtSignature !== undefined); + const signature = signatureIndex < 0 ? undefined : next.content.parts[signatureIndex].thoughtSignature; + if (signature === undefined) return pair; - current.content.parts[targetIndex] = { - ...current.content.parts[targetIndex], - thoughtSignature: signatureOnly.thoughtSignature, - }; - delete signatureOnly.thoughtSignature; - removeRelocatedSignatureParts(next, new Set([signatureOnly]), removedCandidates); - if (next.content.parts.length === 0) { - transferCandidateMetadata(current, next); - delete next.finishReason; - } + const anchored = withParts( + current, + withIndexesChanged(current.content.parts, new Map([[targetIndex, withKeysChanged(current.content.parts[targetIndex], { thoughtSignature: signature })]])), + ); + const strippedParts = keptAfterRelocation( + withIndexesChanged(next.content.parts, new Map([[signatureIndex, withKeysChanged(next.content.parts[signatureIndex], { thoughtSignature: undefined })]])), + new Set([signatureIndex]), + ); + const stripped = withParts(next, strippedParts); + return strippedParts.length > 0 + ? { ...pair, current: anchored, next: stripped } + : { + ...pair, + current: withCandidateMetadataFrom(anchored, stripped), + next: withKeysChanged(stripped, { finishReason: undefined }), + nextRemoved: true, + }; }; -const relocateContinuationSignature = ( - current: GeminiGenerateContentCandidate, - next: GeminiGenerateContentCandidate | undefined, -): void => { - if (next === undefined || current.finishReason !== undefined) return; +const relocateContinuationSignature = (pair: CandidatePair): CandidatePair => { + const { current, next } = pair; + if (next === undefined || current.finishReason !== undefined) return pair; const targetIndex = firstContentIndexOfLastElement(current.content.parts); - const nextPart = firstContentPart(next.content.parts); + const nextIndex = next.content.parts.findIndex(hasPartContent); + const nextPart = nextIndex < 0 ? undefined : next.content.parts[nextIndex]; if ( targetIndex === undefined || nextPart?.thoughtSignature === undefined || !sameLogicalElement(current.content.parts[targetIndex], nextPart) - ) return; - current.content.parts[targetIndex] = { - ...current.content.parts[targetIndex], - thoughtSignature: nextPart.thoughtSignature, + ) return pair; + return { + ...pair, + current: withParts( + current, + withIndexesChanged(current.content.parts, new Map([[targetIndex, withKeysChanged(current.content.parts[targetIndex], { thoughtSignature: nextPart.thoughtSignature })]])), + ), + next: withParts( + next, + withIndexesChanged(next.content.parts, new Map([[nextIndex, withKeysChanged(nextPart, { thoughtSignature: undefined })]])), + ), }; - delete nextPart.thoughtSignature; }; -const removeRelocatedSignatureParts = ( - candidate: GeminiGenerateContentCandidate, - relocated: ReadonlySet, - removedCandidates?: WeakSet, -): void => { - candidate.content.parts = candidate.content.parts.filter(part => - !relocated.has(part) || hasPartContent(part) || part.thoughtSignature !== undefined); - if (candidate.content.parts.length === 0) removedCandidates?.add(candidate); +/** What survives a relocation: a Part it emptied is dropped, and a Part it never touched stays + * whatever it was. Membership is by position rather than by object, because the Parts handed + * in here have already been rebuilt. */ +const keptAfterRelocation = ( + parts: readonly GeminiGenerateContentPart[], + relocated: ReadonlySet, +): readonly GeminiGenerateContentPart[] => { + const kept = parts.filter((part, index) => !relocated.has(index) || hasPartContent(part) || part.thoughtSignature !== undefined); + return kept.length === parts.length ? parts : kept; }; const hasPartContent = (part: GeminiGenerateContentPart): boolean => { @@ -360,51 +438,68 @@ const firstContentIndexOfLastElement = (parts: readonly GeminiGenerateContentPar const firstContentPart = (parts: readonly GeminiGenerateContentPart[]): GeminiGenerateContentPart | undefined => parts.find(hasPartContent); -const mergeEventMetadata = ( +/** Everything an event carries beside its candidates, folded from the two into one — the later + * event's word wins where both have one. The result replaces `target`'s own metadata rather + * than adding to it, so a field neither event set is absent rather than stale. */ +const mergedEventMetadata = ( earlier: GeminiGenerateContentResult, later: GeminiGenerateContentResult, target: GeminiGenerateContentResult, -): void => { +): GeminiGenerateContentResult => { const extras: Record = {}; captureExtras(earlier as unknown as Record, GEMINI_GENERATE_CONTENT_RESULT_KEYS, extras); captureExtras(later as unknown as Record, GEMINI_GENERATE_CONTENT_RESULT_KEYS, extras); const usageMetadata = later.usageMetadata ?? earlier.usageMetadata; const modelVersion = later.modelVersion ?? earlier.modelVersion; const responseId = later.responseId ?? earlier.responseId; - for (const key of Object.keys(target)) if (key !== 'candidates') delete (target as Record)[key]; - Object.assign(target, { + return { + ...(target.candidates !== undefined ? { candidates: target.candidates } : {}), ...(usageMetadata !== undefined ? { usageMetadata } : {}), ...(modelVersion !== undefined ? { modelVersion } : {}), ...(responseId !== undefined ? { responseId } : {}), ...extras, - }); + } as GeminiGenerateContentResult; }; -const clearEventMetadata = (event: GeminiGenerateContentResult): void => { - for (const key of Object.keys(event)) if (key !== 'candidates') delete (event as Record)[key]; -}; +/** The candidates alone. What the event was carrying beside them has moved to its neighbour. */ +const withoutEventMetadata = (event: GeminiGenerateContentResult): GeminiGenerateContentResult => + (event.candidates !== undefined ? { candidates: event.candidates } : {}) as GeminiGenerateContentResult; -const mergeCandidateExtras = ( +/** The same fold one level down, over a candidate's non-standard keys. */ +const mergedCandidateExtras = ( earlier: GeminiGenerateContentCandidate, later: GeminiGenerateContentCandidate, target: GeminiGenerateContentCandidate, -): void => { +): GeminiGenerateContentCandidate => { const extras: Record = {}; captureExtras(earlier as unknown as Record, GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS, extras); captureExtras(later as unknown as Record, GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS, extras); - for (const key of Object.keys(target)) { - if (!GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS.has(key as keyof GeminiGenerateContentCandidate)) delete (target as unknown as Record)[key]; - } - Object.assign(target, extras); + const known = Object.fromEntries( + Object.entries(target).filter(([key]) => GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS.has(key as keyof GeminiGenerateContentCandidate)), + ); + return { ...known, ...extras } as GeminiGenerateContentCandidate; }; -const transferCandidateMetadata = (current: GeminiGenerateContentCandidate, next: GeminiGenerateContentCandidate): void => { - mergeCandidateExtras(current, next, current); - if (next.content.role !== undefined) current.content.role = next.content.role; - if (next.finishReason !== undefined) current.finishReason = next.finishReason; +/** `next` had everything taken out of it, so what it was saying about the turn is said by + * `current` instead. */ +const withCandidateMetadataFrom = ( + current: GeminiGenerateContentCandidate, + next: GeminiGenerateContentCandidate, +): GeminiGenerateContentCandidate => { + const merged = mergedCandidateExtras(current, next, current); + return withKeysChanged(merged, { + ...(next.content.role !== undefined ? { content: { ...merged.content, role: next.content.role } } : {}), + ...(next.finishReason !== undefined ? { finishReason: next.finishReason } : {}), + }); }; -const transferCandidateMetadataForward = (current: GeminiGenerateContentCandidate, next: GeminiGenerateContentCandidate): void => { - mergeCandidateExtras(current, next, next); - if (next.content.role === undefined && current.content.role !== undefined) next.content.role = current.content.role; +/** The same, in the other direction: `current` was emptied into `next`. */ +const withForwardedCandidateMetadata = ( + current: GeminiGenerateContentCandidate, + next: GeminiGenerateContentCandidate, +): GeminiGenerateContentCandidate => { + const merged = mergedCandidateExtras(current, next, next); + return next.content.role === undefined && current.content.role !== undefined + ? withKeysChanged(merged, { content: { ...merged.content, role: current.content.role } }) + : merged; }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts index 116e8448e5..8991fee54c 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts @@ -107,6 +107,12 @@ export class LayeredStatefulOpenAIResponsesStore implements StatefulOpenAIRespon await this.loadItems({ ids: [...ids], itemHashes: [...itemHashes] }); } + // Copied on the way out, which is the one copy in this file the pipeline's own rule asks for + // rather than forbids. What hydration builds from this row is handed to `move()`, so it is + // deep-frozen the moment it becomes a fact — and the row is a live cache entry that outlives + // the turn. Handing the cache's own object over would freeze the cache, which is the same + // hazard `GatewayFacts` states about putting a live handle in the record: the writes the + // store relies on would then fail where nothing is watching. getItemById(id: string): StoredOpenAIResponsesItem | undefined { const row = this.loadedItems.get(id); return row === undefined ? undefined : cloneStoredOpenAIResponsesItem(row); @@ -157,17 +163,22 @@ export class LayeredStatefulOpenAIResponsesStore implements StatefulOpenAIRespon await Promise.all(this.options.writes.map(write => write.insertSnapshot(snapshot))); } + // The scratchpad holds what it was handed. These three used to copy on the way in and again + // on the way out, so that a caller writing into a payload could not reach the store's own — + // but the payloads arriving here came out of the record and are frozen, and the one reader + // splices a sub-object of them into an array rather than writing to it. Copying bought + // isolation from a write nothing performs, at a deep copy per stored item per attempt. beginAttempt(privatePayloads: ReadonlyMap): void { this.privatePayloads.clear(); - for (const [id, payload] of privatePayloads) this.privatePayloads.set(id, structuredClone(payload)); + for (const [id, payload] of privatePayloads) this.privatePayloads.set(id, payload); } registerPrivatePayload(id: string, privatePayload: unknown): void { - if (privatePayload !== undefined) this.privatePayloads.set(id, structuredClone(privatePayload)); + if (privatePayload !== undefined) this.privatePayloads.set(id, privatePayload); } getPrivatePayload(id: string): unknown { - return structuredClone(this.privatePayloads.get(id)); + return this.privatePayloads.get(id); } private async loadItems(query: { ids: readonly string[]; itemHashes: readonly string[] }): Promise { diff --git a/packages/protocols/__tests__/anthropic-messages/usage_test.ts b/packages/protocols/__tests__/anthropic-messages/usage_test.ts index f1eba08643..7475096200 100644 --- a/packages/protocols/__tests__/anthropic-messages/usage_test.ts +++ b/packages/protocols/__tests__/anthropic-messages/usage_test.ts @@ -45,46 +45,31 @@ test('Anthropic Messages usage snapshots merge late counters and atomically repl }); }); -test('Anthropic Messages usage snapshots preserve nullable iterations and isolate all nested iteration data', () => { +test('Anthropic Messages usage snapshots carry the iterations union through, nulls included', () => { + // The array is opaque: nothing in this gateway reads inside an entry, so what a snapshot owes + // it is to arrive whole and to keep saying what the upstream said — including `null`, which is + // the upstream stating it has no iterations rather than not mentioning them. expect(anthropicMessagesUsageSnapshot({ output_tokens: 0, iterations: null })).toEqual({ output_tokens: 0, iterations: null }); - const source = [{ + const iterations = [{ type: 'compaction', input_tokens: 7, cache_creation: { ephemeral_5m_input_tokens: 3 }, - provider_metadata: { - attempts: [{ regions: ['us-east', 'us-west'] }], - }, + provider_metadata: { attempts: [{ regions: ['us-east', 'us-west'] }] }, }]; - const snapshot = anthropicMessagesUsageSnapshot({ output_tokens: 0, iterations: source }); - source[0].cache_creation.ephemeral_5m_input_tokens = 9; - source[0].provider_metadata.attempts[0].regions.push('eu-west'); + const snapshot = anthropicMessagesUsageSnapshot({ output_tokens: 0, iterations }); + expect(snapshot.iterations).toEqual(iterations); - expect(snapshot.iterations).toEqual([{ - type: 'compaction', - input_tokens: 7, - cache_creation: { ephemeral_5m_input_tokens: 3 }, - provider_metadata: { - attempts: [{ regions: ['us-east', 'us-west'] }], - }, - }]); + // A later reading replaces the array rather than merging into it, so a `null` delta is a + // statement and not an omission. expect(mergeAnthropicMessagesUsageSnapshot(snapshot, { output_tokens: 1, iterations: null }).iterations).toBeNull(); }); -test('Anthropic Messages usage snapshot merges isolate opaque nested iteration data from the delta', () => { - const iterations = [{ - type: 'model', - provider_metadata: { - attempts: [{ warnings: ['slow'] }], - }, - }]; - const merged = mergeAnthropicMessagesUsageSnapshot({ output_tokens: 0 }, { output_tokens: 1, iterations }); - iterations[0].provider_metadata.attempts[0].warnings[0] = 'mutated'; +test('Anthropic Messages usage snapshot merges take the delta-s iterations over the current-s', () => { + const current = anthropicMessagesUsageSnapshot({ output_tokens: 0, iterations: [{ type: 'model' }] }); + const iterations = [{ type: 'model', provider_metadata: { attempts: [{ warnings: ['slow'] }] } }]; - expect(merged.iterations).toEqual([{ - type: 'model', - provider_metadata: { - attempts: [{ warnings: ['slow'] }], - }, - }]); + expect(mergeAnthropicMessagesUsageSnapshot(current, { output_tokens: 1, iterations }).iterations).toEqual(iterations); + // And a delta that says nothing about them leaves what the run already had. + expect(mergeAnthropicMessagesUsageSnapshot(current, { output_tokens: 1 }).iterations).toEqual([{ type: 'model' }]); }); diff --git a/packages/protocols/src/anthropic-messages/reassemble.ts b/packages/protocols/src/anthropic-messages/reassemble.ts index c39ebb7c3b..ff7bdac2ec 100644 --- a/packages/protocols/src/anthropic-messages/reassemble.ts +++ b/packages/protocols/src/anthropic-messages/reassemble.ts @@ -12,7 +12,6 @@ import type { AnthropicMessagesUsage, AnthropicMessagesWebSearchToolResultBlock, } from './index.ts'; -import { cloneAnthropicMessagesUsageIterations } from './usage.ts'; import { isJsonObject } from '../common/json.ts'; import { captureExtras } from '../common/reassemble-extras.ts'; @@ -110,9 +109,7 @@ const applyAnthropicMessagesUsage = (usage: AnthropicMessagesUsage, update: Part if (update.server_tool_use != null) { usage.server_tool_use = update.server_tool_use; } - if (update.iterations !== undefined) { - usage.iterations = cloneAnthropicMessagesUsageIterations(update.iterations); - } + if (update.iterations !== undefined) usage.iterations = update.iterations; }; const createBlockAccumulator = (event: Extract): AnthropicMessagesBlockAccumulator => { diff --git a/packages/protocols/src/anthropic-messages/usage.ts b/packages/protocols/src/anthropic-messages/usage.ts index 48eac5cac7..afa38baaff 100644 --- a/packages/protocols/src/anthropic-messages/usage.ts +++ b/packages/protocols/src/anthropic-messages/usage.ts @@ -2,6 +2,12 @@ export interface AnthropicMessagesUsageServerToolUse { web_search_requests?: number; } +// The beta usage union includes model attempts, advisor attempts, and +// compaction entries, and remains additively extensible. Floway carries it +// opaquely: nothing reads inside an entry and nothing writes one, so the array +// an upstream sent travels by reference from the event to the snapshot rather +// than being deep-copied at each of the three hops that touch it. +// https://github.com/anthropics/anthropic-sdk-typescript/blob/3b45cd3b69c956ac63384fdb09ce1d8109f3fa80/src/resources/beta/messages/messages.ts#L1724-L1829 export interface AnthropicMessagesUsageIteration { type: string; model?: string; @@ -16,13 +22,6 @@ export interface AnthropicMessagesUsageIteration { [key: string]: unknown; } -// The beta usage union includes model attempts, advisor attempts, and -// compaction entries, and remains additively extensible. Floway only needs an -// isolated opaque snapshot, not a closed projection of those variants. -// https://github.com/anthropics/anthropic-sdk-typescript/blob/3b45cd3b69c956ac63384fdb09ce1d8109f3fa80/src/resources/beta/messages/messages.ts#L1724-L1829 -export const cloneAnthropicMessagesUsageIterations = (iterations: AnthropicMessagesUsageIteration[] | null): AnthropicMessagesUsageIteration[] | null => - iterations === null ? null : structuredClone(iterations); - export interface AnthropicMessagesUsage { input_tokens: number; output_tokens: number; @@ -74,7 +73,6 @@ export const anthropicMessagesUsageSnapshot = (usage?: AnthropicMessagesUsageSna ...usage, ...(usage.cache_creation === undefined ? {} : { cache_creation: { ...usage.cache_creation } }), ...(usage.output_tokens_details === undefined ? {} : { output_tokens_details: { ...usage.output_tokens_details } }), - ...(usage.iterations === undefined ? {} : { iterations: cloneAnthropicMessagesUsageIterations(usage.iterations) }), }; export const mergeAnthropicMessagesUsageSnapshot = ( @@ -88,7 +86,7 @@ export const mergeAnthropicMessagesUsageSnapshot = ( ...(delta.cache_creation_input_tokens === undefined ? {} : { cache_creation_input_tokens: delta.cache_creation_input_tokens }), ...(delta.cache_creation === undefined ? {} : { cache_creation: { ...delta.cache_creation } }), ...(delta.output_tokens_details === undefined ? {} : { output_tokens_details: { ...delta.output_tokens_details } }), - ...(delta.iterations === undefined ? {} : { iterations: cloneAnthropicMessagesUsageIterations(delta.iterations) }), + ...(delta.iterations === undefined ? {} : { iterations: delta.iterations }), ...(delta.speed === undefined && delta.service_tier === undefined ? {} : { speed: delta.speed, service_tier: delta.service_tier }), From 0ab3c0caf2032af0a1a85fb60c4ae70764d16ad8 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 16:54:35 +0800 Subject: [PATCH 103/156] refactor(openai-responses): rebuild the affinity payload the way its siblings do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fourth materializer. It never cloned, so the clone sweep passed over it — it built a mutable draft with `{ ...item }` and wrote the slots into that, which is correct but is the one of four that says it differently. An item no projection touches now rides through by identity rather than becoming a fresh copy, and the remove branch is `undefined` in a patch rather than a `delete` on a draft. Same helpers as the other three, so the family reads as one thing. --- .../chat/openai-responses/affinity/ingress.ts | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/openai-responses/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/openai-responses/affinity/ingress.ts index 63378879dc..f8c3465719 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/affinity/ingress.ts @@ -9,6 +9,7 @@ import { projectOptionalAffinityBlob, projectRequiredAffinityBlob, } from '../../shared/affinity/index.ts'; +import { withIndexesChanged, withKeysChanged } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; import type { ModelCandidate } from '@floway-dev/provider'; @@ -134,36 +135,34 @@ const materializeOpenAIResponsesPayload = ( projectionsByItem: ReadonlyMap, ): CanonicalOpenAIResponsesPayload => { if (projectionsByItem.size === 0) return payload; + // Rebuilt the way its three sibling families are: an item no projection touches rides through + // by identity, and what one candidate is owed differs from the next by the slots a projection + // actually spoke about. const input = payload.input.flatMap((item, itemIndex): OpenAIResponsesInputItem[] => { const projections = projectionsByItem.get(itemIndex); if (projections === undefined) return [item]; if (projections === null) return []; - const replacement = { ...item } as OpenAIResponsesInputItem & Record; - for (const { location, projection } of projections) { - if (location.contentIndex !== undefined) continue; - if (projection.kind === 'preserve') replacement[location.slot] = projection.value; - else delete replacement[location.slot]; - } - - if (item.type === 'agent_message') { - const nested = new Map(projections.flatMap(projection => - projection.location.contentIndex === undefined ? [] : [[projection.location.contentIndex, projection] as const])); - if (nested.size > 0) { - const agentMessage = replacement as Extract; - agentMessage.content = agentMessage.content.flatMap((content, contentIndex) => { - const projected = nested.get(contentIndex); - if (projected === undefined) return [content]; - return projected.projection.kind === 'preserve' - ? [{ ...content, encrypted_content: projected.projection.value }] - : []; - }); - } - } - - return [replacement]; + const slots = Object.fromEntries(projections.flatMap(({ location, projection }) => + location.contentIndex !== undefined + ? [] + : [[location.slot, projection.kind === 'preserve' ? projection.value : undefined] as const])); + const rewritten = withKeysChanged(item, slots); + if (item.type !== 'agent_message') return [rewritten]; + + const content = (rewritten as Extract).content; + const nested = new Map(projections.flatMap(({ location, projection }) => + location.contentIndex === undefined + ? [] + : [[ + location.contentIndex, + projection.kind === 'preserve' + ? { ...content[location.contentIndex], encrypted_content: projection.value } + : null, + ] as const])); + return nested.size === 0 ? [rewritten] : [withKeysChanged(rewritten, { content: withIndexesChanged(content, nested) })]; }); - return { ...payload, input }; + return withKeysChanged(payload, { input }); }; const evaluateOpenAIResponsesCandidate = ( From 81f8cef375dda4137b2c7b54c61d0e298541bc1e Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 18:23:39 +0800 Subject: [PATCH 104/156] refactor(gateway): render every family's failure through the same three tiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chat families answer a failure through `renderFailure`; the five non-chat families called `renderErrorEnvelope(message, body)` directly and wrote `answer.status` beside it. Two expressions of one concept, and the one they used has two tiers where the mechanism has three — a `Failure` carrying `envelope`, the refusal a stage wrote in its own words, was silently dropped for `/v1/completions`, `/v1/embeddings`, rerank, images and audio. Nothing sets `envelope` in those families today, so this fixes no live defect. It closes the way one would arrive: the tier is optional on the type, so the next stage to write a refusal with a `code` and a `param` would have found them gone with nothing failing. Now every family answers through the same function, and body and status come from it together. `renderErrorEnvelope` stays what it always was — the OpenAI-shaped envelope, in the protocols package — reached through `mintedErrorEnvelope` as the third tier rather than called as the whole policy. --- .../src/data-plane/openai-audio/pipeline.ts | 9 ++++--- .../data-plane/openai-completions/pipeline.ts | 24 ++++++++++++------ .../data-plane/openai-embeddings/pipeline.ts | 9 ++++--- .../src/data-plane/openai-images/pipeline.ts | 25 +++++++++++++------ .../gateway/src/data-plane/rerank/pipeline.ts | 19 +++++++++----- 5 files changed, 56 insertions(+), 30 deletions(-) diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index 24863d1bbe..44257d6d06 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -16,7 +16,7 @@ import { recordStream, streamReferenceOf } from '../../dump/run-sink.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; @@ -25,7 +25,7 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/te import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { compose, defer, defineStage, move, own, type Deferred, type Logger, type Owned, type Pipeline } from '@floway-dev/pipeline'; -import { eventFrame, isEventStreamMediaType, parseDecimalString, parseSSEStream, renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { eventFrame, isEventStreamMediaType, parseDecimalString, parseSSEStream, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; import { isOpenAIAudioTranscriptionDoneEvent, parseOpenAIAudioTranscription, @@ -146,12 +146,13 @@ const emitOpenAIAudioTranscription = defineStage< const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); return { ...rest, 'response.http.headers': forClient, - 'response.http.status': answer.status, + 'response.http.status': failure.status, 'response.openaiAudioTranscription.mediaType': 'application/json', - 'response.openaiAudioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.openaiAudioTranscription.rendered': move(failure.body), }; } // Everything that reaches here answered. The same key carried the upstream's own status diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index ae38f41204..e9d249abe2 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -17,7 +17,7 @@ import { recordStream, streamReferenceOf, type RunDump } from '../../dump/run-si import type { UsageQuantities } from '../../repo/types.ts'; import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; @@ -27,7 +27,7 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/te import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { compose, defer, defineStage, move, own, type Deferred, type Owned, type Pipeline } from '@floway-dev/pipeline'; -import { isOpenAIUsageOnlyEventShape, renderErrorEnvelope, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { openaiCompletionsProtocolFrameToSSEFrame, parseOpenAICompletionsResult, @@ -125,25 +125,33 @@ const emitOpenAICompletions = defineStage< // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); // A refusal keeps the status the upstream gave it. Anything that answered is a 200 the // gateway says itself, because what the client receives is serialized here rather than // relayed — the upstream's own status is a fact further down for whoever wants it. + if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); + return { + ...rest, + 'response.http.headers': forClient, + 'response.http.status': failure.status, + 'response.openaiCompletions.rendered': move(failure.body), + }; + } return { ...rest, - 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), - 'response.http.status': isFailure(answer) ? answer.status : 200, + 'response.http.headers': forClient, + 'response.http.status': 200, 'response.openaiCompletions.rendered': move(rendered(answer, back['ingress.openaiCompletions.wantsUsageChunk'])), }; }, }); const rendered = ( - answer: OpenAICompletionsFacts['response.openaiCompletions.payload'], + answer: Exclude, wantsUsageChunk: boolean, ): OpenAICompletionsFacts['response.openaiCompletions.rendered'] => - isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) - : isFrames(answer) ? renderSSE(answer, wantsUsageChunk) - : answer; + isFrames(answer) ? renderSSE(answer, wantsUsageChunk) : answer; const renderSSE = (frames: OpenAICompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => ({ // The frames the client reads are a reframing of the ones the record holds, so this key diff --git a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts index 3ed13760b5..a787afcc51 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts @@ -13,7 +13,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates, type Narrowing } from '../pipeline/stages.ts'; @@ -23,7 +23,7 @@ import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import type { Pipeline } from '@floway-dev/pipeline'; import { compose, defineStage, move } from '@floway-dev/pipeline'; -import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage } from '@floway-dev/protocols/common'; +import { parseDecimalString, upstreamErrorMessage } from '@floway-dev/protocols/common'; import { parseOpenAIEmbeddingsResponse, renderOpenAIEmbeddingsResponse, @@ -85,14 +85,15 @@ const emitOpenAIEmbeddings = defineStage< const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); return { ...rest, 'response.http.headers': forClient, - 'response.openaiEmbeddings.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.openaiEmbeddings.rendered': move(failure.body), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what // happened, and a 429 arriving as a 200 is not that. - 'response.http.status': answer.status, + 'response.http.status': failure.status, }; } return { diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index da16894e84..6190ff351a 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -14,7 +14,7 @@ import { recordStream, streamReferenceOf } from '../../dump/run-sink.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; @@ -29,7 +29,6 @@ import { isEventStreamMediaType, mediaTypeEssence, parseDecimalString, - renderErrorEnvelope, upstreamErrorMessage, type ModelEndpointKey, type SseFrame, @@ -137,19 +136,29 @@ const emitOpenAIImages = defineStage< // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); + return { + ...rest, + 'response.http.headers': forClient, + 'response.http.status': failure.status, + 'response.openaiImages.rendered': move(failure.body), + }; + } return { ...rest, - 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), - 'response.http.status': isFailure(answer) ? answer.status : 200, + 'response.http.headers': forClient, + 'response.http.status': 200, 'response.openaiImages.rendered': move(rendered(answer)), }; }, }); -const rendered = (answer: OpenAIImagesFacts['response.openaiImages.canonical']): OpenAIImagesFacts['response.openaiImages.rendered'] => - isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) - : isFrames(answer) ? renderSSE(answer) - : renderOpenAIImagesResponse(answer); +const rendered = ( + answer: Exclude, +): OpenAIImagesFacts['response.openaiImages.rendered'] => + isFrames(answer) ? renderSSE(answer) : renderOpenAIImagesResponse(answer); const renderSSE = (frames: OpenAIImagesFrames): AsyncIterable => ({ // The frames the client reads are a reframing of the ones the record holds, so this key diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index 17e62b8cf3..3cf49e2b01 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -12,7 +12,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; @@ -22,7 +22,7 @@ import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import type { Pipeline } from '@floway-dev/pipeline'; import { defineStage, move, compose } from '@floway-dev/pipeline'; -import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage, type RerankProtocol, type RerankSourceProtocol } from '@floway-dev/protocols/common'; +import { parseDecimalString, upstreamErrorMessage, type RerankProtocol, type RerankSourceProtocol } from '@floway-dev/protocols/common'; import { parseRerankResponse, parseRerankUsage, @@ -85,14 +85,15 @@ const emitRerank = defineStage< const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); return { ...rest, 'response.http.headers': forClient, - 'response.rerank.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.rerank.rendered': move(failure.body), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what // happened, and a 429 arriving as a 200 is not that. - 'response.http.status': answer.status, + 'response.http.status': failure.status, }; } // Translating can fail on an answer that parsed: a result may index a document the @@ -108,11 +109,17 @@ const emitRerank = defineStage< back['request.rerank.canonical'], ); } catch (error) { + // The gateway's own failure, through the same three tiers as any other: it has neither an + // upstream body nor an envelope of its own, so what a client is sent is the mint. + const failure = renderFailure( + { status: 502, message: error instanceof Error ? error.message : String(error) }, + mintedErrorEnvelope, + ); return { ...rest, 'response.http.headers': forClient, - 'response.rerank.rendered': move(renderErrorEnvelope(error instanceof Error ? error.message : String(error))), - 'response.http.status': 502, + 'response.rerank.rendered': move(failure.body), + 'response.http.status': failure.status, }; } return { From 90e85942fa0883c8903861969f54c3e9c98c087f Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 18:34:15 +0800 Subject: [PATCH 105/156] refactor(protocols): leave the three-tier decision to the one thing that makes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `renderProtocolError` was the forward-or-mint rule, and `renderErrorEnvelope` wrapped it so a caller could pass an upstream body and get either. With every family now answering through `renderFailure`, the gateway holds that rule in one place and passes no body here — so the wrapper's second parameter had no caller and the rule underneath it was a second copy of a decision already made. What is left is what this package actually owns: the shape an OpenAI-shaped protocol states a gateway-authored refusal in. The test follows — it was asserting the tiering through this seam, and the tiering is tested where it now lives. --- .../__tests__/openai-images/response_test.ts | 7 ++++--- .../protocols/src/common/error-envelope.ts | 21 ++++++------------- packages/protocols/src/common/index.ts | 2 +- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/packages/protocols/__tests__/openai-images/response_test.ts b/packages/protocols/__tests__/openai-images/response_test.ts index d1b590b086..4fef2fe02d 100644 --- a/packages/protocols/__tests__/openai-images/response_test.ts +++ b/packages/protocols/__tests__/openai-images/response_test.ts @@ -63,14 +63,15 @@ describe('OpenAI Images usage', () => { }); describe('OpenAI Images failures', () => { - test('an upstream error object is what the client is sent, and anything else becomes the gateway envelope', () => { + test('an upstream error object yields its message, and the gateway envelope carries the gateway-s', () => { + // Which of the two a client is actually sent is `renderFailure`'s decision, over all three + // tiers at once; what this protocol owns is reading an upstream's words and writing its own. const upstream = { error: { message: 'Your request was rejected', type: 'image_generation_user_error', code: 'moderation_blocked' } }; expect(upstreamErrorMessage(upstream)).toBe('Your request was rejected'); - expect(renderErrorEnvelope('Your request was rejected', upstream)).toBe(upstream); expect(upstreamErrorMessage('upstream is down')).toBeUndefined(); - expect(renderErrorEnvelope('Model gpt-image-1 is not available on any configured upstream.', undefined)) + expect(renderErrorEnvelope('Model gpt-image-1 is not available on any configured upstream.')) .toEqual({ error: { message: 'Model gpt-image-1 is not available on any configured upstream.', type: 'api_error' } }); }); }); diff --git a/packages/protocols/src/common/error-envelope.ts b/packages/protocols/src/common/error-envelope.ts index 04b74c73c9..222b60d551 100644 --- a/packages/protocols/src/common/error-envelope.ts +++ b/packages/protocols/src/common/error-envelope.ts @@ -24,19 +24,10 @@ export const upstreamErrorMessage = (body: unknown): string | undefined => { return typeof body.error.message === 'string' ? body.error.message : undefined; }; -/** - * The rule, with the envelope left to the protocol that writes it. +/** The envelope the OpenAI-shaped protocols write when a refusal is the gateway's own words. * - * An upstream that refused in its own words is handed on in them whatever protocol it spoke, - * because those words are already the shape its own client reads. Only a refusal the gateway - * itself produced has no body to forward, and that is the one case where which protocol the - * *client* speaks decides the shape. - */ -export const renderProtocolError = ( - upstreamBody: unknown, - envelope: () => Record, -): Record => isJsonObject(upstreamBody) ? upstreamBody : envelope(); - -/** The envelope the OpenAI-shaped protocols write. */ -export const renderErrorEnvelope = (message: string, upstreamBody?: unknown): Record => - renderProtocolError(upstreamBody, () => ({ error: { message, type: 'api_error' } })); + * Which of an upstream's body, a stage's own envelope and this one a client is actually sent is + * not decided here: that is one policy over all three tiers, and it lives with the `Failure` it + * reads. This is only the shape those protocols state a gateway-authored refusal in. */ +export const renderErrorEnvelope = (message: string): Record => + ({ error: { message, type: 'api_error' } }); diff --git a/packages/protocols/src/common/index.ts b/packages/protocols/src/common/index.ts index cc622541fa..f146029528 100644 --- a/packages/protocols/src/common/index.ts +++ b/packages/protocols/src/common/index.ts @@ -13,7 +13,7 @@ export * from './sse.ts'; export * from './parse-sse.ts'; export * from './parse-events.ts'; -export { renderErrorEnvelope, renderProtocolError, upstreamErrorMessage } from './error-envelope.ts'; +export { renderErrorEnvelope, upstreamErrorMessage } from './error-envelope.ts'; export { isJsonObject, type JsonObject } from './json.ts'; export { captureExtras } from './reassemble-extras.ts'; export { mapKeepingIdentity, withIndexesChanged, withKeysChanged } from './rebuild.ts'; From 0fa35a17eee06e40f86f7d5c0d628e0b22f09912 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 18:58:12 +0800 Subject: [PATCH 106/156] refactor(openai-responses): keep the sub-request pipelines inside their own modules Thirteen of the fifteen composed pipelines are exported because an http module mounts them. These two are not: each is reached only by the `run*SubRequest` beside it, so the export said "public" about a seam nothing outside can use. Found by asking which composed pipelines are reachable from outside the file that composes them, rather than by counting how many are exported. --- .../chat/openai-responses/server-tools/image-sub-request.ts | 2 +- .../chat/openai-responses/server-tools/search-sub-request.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts index 6e378020f3..4b81655147 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts @@ -76,7 +76,7 @@ const dialImageGeneration = defineStage< }, }); -export const imageGenerationSubRequestPipeline: Pipeline< +const imageGenerationSubRequestPipeline: Pipeline< I<'request.imageGeneration.call'>, I<'response.imageGeneration.lifecycle' | 'response.imageGeneration.streamedUsage'> > = compose('imageGenerationSubRequest', [ diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts index f8cd50c304..09eb751508 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts @@ -46,7 +46,7 @@ const runWebSearchCall = defineStage< }) as never, }); -export const webSearchSubRequestPipeline: Pipeline, W<'response.webSearch.ir'>> = +const webSearchSubRequestPipeline: Pipeline, W<'response.webSearch.ir'>> = compose('webSearchSubRequest', [ writeSettlement(() => false), runWebSearchCall, From f70d833359d744e54a1723d23c2c87b21e23e28e Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 19:34:22 +0800 Subject: [PATCH 107/156] refactor(interceptor): drop the ambient slot the gateway's chains needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Interceptor` had a second slot for what surrounds a call, and the module said what it was for: "the gateway's chat chains pass the request-scoped gateway context there, while the provider-boundary chains have nothing ambient to hand down and pass `{}`". Those chat chains are stages now, so all eight call sites pass `object` and every interceptor names the parameter `_env`. The slot outlived its only user. It is gone, along with the twenty-three `_env: object` parameters and the `stubRequest = {}` fixtures that existed to fill it. The header says what the framework is now: one slot, provider boundaries only. The mutation convention stays, and now says which side of the house it belongs to. A boundary shapes one wire call it owns outright and writes into `ctx`; a turn travelling through the gateway is a fact record, frozen at every handover. Both are true, and the comment used to describe only the first while the second was being built. The codemod over-applied twice — `, {}, ` is an ordinary argument in `packages/translate` and in five gateway tests (import data, a search body, a Hono env, a usage map). Both reverted after reading the diff against the files; what is left is the interceptor pattern only. --- AGENTS.md | 2 +- packages/interceptor/__tests__/index_test.ts | 19 ++++---- packages/interceptor/src/index.ts | 44 ++++++++++--------- .../fill-empty-namespace-descriptions_test.ts | 2 +- .../fill-empty-namespace-descriptions.ts | 1 - .../interceptors/openai-responses/types.ts | 1 - packages/provider-azure/src/provider.ts | 4 +- .../backfill-required-fields_test.ts | 12 ++--- .../hoist-user-system-to-messages_test.ts | 8 ++-- .../inject-billing-block_test.ts | 4 +- .../inject-default-template_test.ts | 8 ++-- .../inject-identity-block_test.ts | 4 +- .../synthesize-metadata-user-id_test.ts | 20 ++++----- .../backfill-required-fields.ts | 1 - .../hoist-user-system-to-messages.ts | 1 - .../interceptors/anthropic-messages/index.ts | 2 +- .../inject-billing-block.ts | 1 - .../inject-default-template.ts | 1 - .../inject-identity-block.ts | 1 - .../synthesize-metadata-user-id.ts | 1 - packages/provider-claude-code/src/provider.ts | 3 +- .../openai-responses/action-pivot_test.ts | 2 +- .../inject-default-instructions_test.ts | 14 +++--- .../strip-unsupported-fields_test.ts | 8 ++-- .../provider-codex/__tests__/provider_test.ts | 2 +- .../interceptors/openai-responses/index.ts | 2 +- .../inject-default-instructions.ts | 1 - .../strip-unsupported-fields.ts | 1 - packages/provider-codex/src/provider.ts | 4 +- .../apply-top-level-cache-control_test.ts | 14 +++--- .../anthropic-messages/boundary-chain_test.ts | 5 +-- .../compress-images_test.ts | 12 +++-- .../handle-speed-fast_test.ts | 18 ++++---- .../normalize-anthropic-beta_test.ts | 11 +++-- .../promote-thinking-display_test.ts | 16 +++---- .../set-claude-agent-headers_test.ts | 14 +++--- .../set-compact-headers_test.ts | 26 +++++------ .../set-initiator-header_test.ts | 10 ++--- .../set-interaction-id-header_test.ts | 18 ++++---- .../set-vision-header_test.ts | 8 ++-- .../strip-cache-control-extensions_test.ts | 8 ++-- .../abort-on-tool-argument-whitespace_test.ts | 4 +- .../attach-cache-control-markers_test.ts | 16 +++---- .../compress-images_test.ts | 8 ++-- .../set-initiator-header_test.ts | 10 ++--- .../set-vision-header_test.ts | 6 +-- .../abort-on-tool-argument-whitespace_test.ts | 4 +- .../openai-responses/action-pivot_test.ts | 2 +- .../openai-responses/compress-images_test.ts | 14 +++--- .../fill-empty-namespace-descriptions_test.ts | 2 +- .../force-store-false_test.ts | 8 ++-- .../openai-responses/item-id-membrane_test.ts | 8 ++-- .../set-initiator-header_test.ts | 16 +++---- .../set-vision-header_test.ts | 6 +-- .../apply-top-level-cache-control.ts | 2 +- .../anthropic-messages/compress-images.ts | 1 - .../anthropic-messages/handle-speed-fast.ts | 2 +- .../normalize-anthropic-beta.ts | 1 - .../promote-thinking-display.ts | 2 +- .../rewrite-context-window-error.ts | 2 +- .../set-claude-agent-headers.ts | 2 +- .../anthropic-messages/set-compact-headers.ts | 2 +- .../set-initiator-header.ts | 1 - .../set-interaction-id-header.ts | 2 +- .../anthropic-messages/set-vision-header.ts | 1 - .../strip-cache-control-extensions.ts | 2 +- .../strip-eager-input-streaming.ts | 2 +- .../interceptors/anthropic-messages/types.ts | 2 - .../abort-on-tool-argument-whitespace.ts | 2 +- .../attach-cache-control-markers.ts | 2 +- .../compress-images.ts | 2 +- .../set-initiator-header.ts | 2 +- .../set-vision-header.ts | 2 +- .../openai-chat-completions/types.ts | 1 - .../abort-on-tool-argument-whitespace.ts | 2 +- .../openai-responses/compress-images.ts | 1 - .../fill-empty-namespace-descriptions.ts | 1 - .../openai-responses/force-store-false.ts | 1 - .../openai-responses/item-id-membrane.ts | 2 +- .../openai-responses/set-initiator-header.ts | 1 - .../openai-responses/set-vision-header.ts | 1 - .../strip-image-generation.ts | 1 - .../openai-responses/strip-service-tier.ts | 1 - .../interceptors/openai-responses/types.ts | 1 - packages/provider-copilot/src/provider.ts | 16 +++---- 85 files changed, 214 insertions(+), 287 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8bde1508ed..48921f82d8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ | Package | `packages/agent-setup` | Configures supported coding agents. | | Package | `packages/gateway` | Composes gateway services. | | Package | `packages/http` | Provides HTTP transport primitives. | -| Package | `packages/interceptor` | Intercepts gateway traffic. | +| Package | `packages/interceptor` | Chains provider boundary rewrites. | | Package | `packages/pipeline` | Runs stages over a fact record. | | Package | `packages/platform` | Defines portable runtime contracts. | | Package | `packages/protocols` | Defines protocol contracts. | diff --git a/packages/interceptor/__tests__/index_test.ts b/packages/interceptor/__tests__/index_test.ts index c638f43dad..3453838fec 100644 --- a/packages/interceptor/__tests__/index_test.ts +++ b/packages/interceptor/__tests__/index_test.ts @@ -4,25 +4,24 @@ import { type Interceptor, runInterceptors } from '../src/index.ts'; import { assertEquals, assertRejects } from '@floway-dev/test-utils'; type TestCtx = { payload: { value: string } }; -type TestEnv = { traceId: string }; test('composes interceptors outermost-first and unwinds epilogues inside-out', async () => { const calls: string[] = []; - const outer: Interceptor = async (_ctx, _env, run) => { + const outer: Interceptor = async (_ctx, run) => { calls.push('outer-before'); const result = await run(); calls.push('outer-after'); return result; }; - const inner: Interceptor = async (_ctx, _env, run) => { + const inner: Interceptor = async (_ctx, run) => { calls.push('inner-before'); const result = await run(); calls.push('inner-after'); return result; }; - await runInterceptors({ payload: { value: 'ok' } }, { traceId: 't' }, [outer, inner], () => { + await runInterceptors({ payload: { value: 'ok' } }, [outer, inner], () => { calls.push('terminal'); return Promise.resolve('done'); }); @@ -34,14 +33,14 @@ test('lets an interceptor retry by calling run() again — each call reruns the const ctx: TestCtx = { payload: { value: 'broken' } }; let attempts = 0; - const interceptor: Interceptor = async (current, _env, run) => { + const interceptor: Interceptor = async (current, run) => { const first = await run(); if (first !== 'fail') return first; current.payload.value = 'fixed'; return await run(); }; - const result = await runInterceptors(ctx, { traceId: 't' }, [interceptor], () => { + const result = await runInterceptors(ctx, [interceptor], () => { attempts += 1; return Promise.resolve(ctx.payload.value === 'broken' ? 'fail' : ctx.payload.value); }); @@ -53,7 +52,7 @@ test('lets an interceptor retry by calling run() again — each call reruns the test('propagates an inner throw past each enclosing run() call site without swallowing', async () => { const seen: string[] = []; - const wrap = (label: string): Interceptor => async (_ctx, _env, run) => { + const wrap = (label: string): Interceptor => async (_ctx, run) => { seen.push(`${label}-before`); try { return await run(); @@ -64,7 +63,7 @@ test('propagates an inner throw past each enclosing run() call site without swal const boom = new Error('upstream blew up'); await assertRejects( - () => runInterceptors({ payload: { value: 'x' } }, { traceId: 't' }, [wrap('outer'), wrap('inner')], () => Promise.reject(boom)), + () => runInterceptors({ payload: { value: 'x' } }, [wrap('outer'), wrap('inner')], () => Promise.reject(boom)), Error, 'upstream blew up', ); @@ -75,13 +74,13 @@ test('propagates an inner throw past each enclosing run() call site without swal test('lets an interceptor patch context before run and transform the result after run', async () => { const ctx: TestCtx = { payload: { value: 'original' } }; - const interceptor: Interceptor = async (current, _env, run) => { + const interceptor: Interceptor = async (current, run) => { current.payload.value = 'patched'; const result = await run(); return `${result}:${current.payload.value}`; }; - const result = await runInterceptors(ctx, { traceId: 't' }, [interceptor], () => Promise.resolve(ctx.payload.value)); + const result = await runInterceptors(ctx, [interceptor], () => Promise.resolve(ctx.payload.value)); assertEquals(ctx.payload.value, 'patched'); assertEquals(result, 'patched:patched'); diff --git a/packages/interceptor/src/index.ts b/packages/interceptor/src/index.ts index 7c56eea7e2..fe156d1794 100644 --- a/packages/interceptor/src/index.ts +++ b/packages/interceptor/src/index.ts @@ -1,29 +1,32 @@ // Interceptors wrap a single typed call. Each interceptor receives the call's -// own invocation state, the ambient environment around it, and a `run` to -// delegate to the next interceptor (the innermost run executes the call -// itself). Interceptors may inspect or mutate that invocation state before -// `run`, await `run` and transform the result, short-circuit by returning -// without calling `run`, or retry by invoking `run` again. The shape is -// intentionally generic in Ctx/Env/Result so it works for any kind of call — -// provider-side wire shaping, source-side translation, retry policy — wired by -// the caller into concrete chains. +// own invocation state and a `run` to delegate to the next interceptor (the +// innermost run executes the call itself). Interceptors may inspect or mutate +// that invocation state before `run`, await `run` and transform the result, +// short-circuit by returning without calling `run`, or retry by invoking `run` +// again. The shape is generic in Ctx/Result so it works for any kind of call, +// wired by the caller into concrete chains. // // `Ctx` carries the call itself — payload, headers, chosen target — and is the -// slot interceptors write to. `Env` carries what surrounds the call and does -// not belong to it: the gateway's chat chains pass the request-scoped gateway -// context there, while the provider-boundary chains have nothing ambient to -// hand down and pass `{}`. +// slot interceptors write to. There is no ambient second slot: every chain this +// runs is a provider boundary chain, and a boundary has nothing around the call +// to hand down. // // ## Mutation convention // -// Mutations applied to `ctx` or `env` before `run()` propagate forward through -// every downstream interceptor and into the terminal call. They are **one-way**: -// the interceptor that wrote a field does not restore it on the way out, and -// the framework does not snapshot/rewind state for it. Whatever consumes the +// Mutations applied to `ctx` before `run()` propagate forward through every +// downstream interceptor and into the terminal call. They are **one-way**: the +// interceptor that wrote a field does not restore it on the way out, and the +// framework does not snapshot/rewind state for it. Whatever consumes the // chain's output post-run (the caller that invoked `runInterceptors`, an outer // interceptor's after-`run()` code) must keep its own captured copy of any input // it still needs. // +// This is a provider-boundary convention and not the gateway's: what a request +// travels in through the gateway is a fact record, which is frozen at every +// handover and rebuilt rather than written to. The two live side by side because +// they answer different questions — a boundary shapes one wire call it owns +// outright, a pipeline carries a turn nobody owns alone. +// // The convention exists because partial adoption is the worst case: if some // interceptors restore and others don't, there is no honest invariant the // rest of the codebase can rely on — readers can no longer tell what `ctx` @@ -35,14 +38,13 @@ // writes `ctx.foo = bar` in `try` and `ctx.foo = original` in `finally` is a // convention violation, not a feature. export type InterceptorRun = () => Promise; -export type Interceptor = (ctx: Ctx, env: Env, run: InterceptorRun) => Promise; +export type Interceptor = (ctx: Ctx, run: InterceptorRun) => Promise; -export const runInterceptors = async ( +export const runInterceptors = async ( ctx: Ctx, - env: Env, - interceptors: readonly Interceptor[], + interceptors: readonly Interceptor[], terminal: InterceptorRun, ): Promise => { - const run = (index: number): Promise => (index < interceptors.length ? interceptors[index](ctx, env, () => run(index + 1)) : terminal()); + const run = (index: number): Promise => (index < interceptors.length ? interceptors[index](ctx, () => run(index + 1)) : terminal()); return await run(0); }; diff --git a/packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts b/packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts index 56854cf36d..5a218c453b 100644 --- a/packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts +++ b/packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts @@ -36,7 +36,7 @@ test('fills Codex OpenAI Responses Lite namespace descriptions before Azure disp }], }); - await withEmptyNamespaceDescriptionsFilled(ctx, {}, async () => {}); + await withEmptyNamespaceDescriptionsFilled(ctx, async () => {}); const [item] = ctx.payload.input; if (item?.type !== 'additional_tools') throw new Error('expected additional_tools input'); diff --git a/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts b/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts index b44ac95749..84603d8c12 100644 --- a/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts +++ b/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts @@ -12,7 +12,6 @@ import { mapOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses' */ export const withEmptyNamespaceDescriptionsFilled = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { ctx.payload = mapOpenAIResponsesTools(ctx.payload, tool => tool.type === 'namespace' && tool.description === '' diff --git a/packages/provider-azure/src/interceptors/openai-responses/types.ts b/packages/provider-azure/src/interceptors/openai-responses/types.ts index 13747397ff..776def1da3 100644 --- a/packages/provider-azure/src/interceptors/openai-responses/types.ts +++ b/packages/provider-azure/src/interceptors/openai-responses/types.ts @@ -11,6 +11,5 @@ export interface OpenAIResponsesBoundaryCtx { export type AzureOpenAIResponsesBoundaryInterceptor = Interceptor< OpenAIResponsesBoundaryCtx, - object, ProviderOpenAIResponsesResult >; diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index b46fd1e28b..3470add474 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -73,8 +73,8 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { model, action, }; - return await runInterceptors( - ctx, {}, AZURE_OPENAI_RESPONSES_BOUNDARY, async () => { + return await runInterceptors( + ctx, AZURE_OPENAI_RESPONSES_BOUNDARY, async () => { const { model: _ignored, ...wireBody } = ctx.payload; switch (ctx.action) { case 'generate': { diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts index fdf3e5090e..93192c87d9 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts +++ b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts @@ -26,7 +26,7 @@ test('backfills max_tokens from ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS when both temperature: 0.5, } as Partial as AnthropicMessagesPayload); - await backfillRequiredFields(ctx, {}, okEvents); + await backfillRequiredFields(ctx, okEvents); assertEquals(ctx.payload.max_tokens, ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS); }); @@ -42,7 +42,7 @@ test('prefers model.limits.max_output_tokens over the gateway fallback when set' model, ); - await backfillRequiredFields(ctx, {}, okEvents); + await backfillRequiredFields(ctx, okEvents); assertEquals(ctx.payload.max_tokens, 64000); }); @@ -59,7 +59,7 @@ test('preserves caller-supplied max_tokens', async () => { model, ); - await backfillRequiredFields(ctx, {}, okEvents); + await backfillRequiredFields(ctx, okEvents); assertEquals(ctx.payload.max_tokens, 100); }); @@ -74,7 +74,7 @@ test('backfills temperature to 1 when missing', async () => { messages: [{ role: 'user', content: 'hi' }], }); - await backfillRequiredFields(ctx, {}, okEvents); + await backfillRequiredFields(ctx, okEvents); assertEquals(ctx.payload.temperature, 1); }); @@ -87,7 +87,7 @@ test('preserves caller-supplied temperature', async () => { temperature: 0.7, }); - await backfillRequiredFields(ctx, {}, okEvents); + await backfillRequiredFields(ctx, okEvents); assertEquals(ctx.payload.temperature, 0.7); }); @@ -102,7 +102,7 @@ test('preserves caller-supplied temperature: 0', async () => { temperature: 0, }); - await backfillRequiredFields(ctx, {}, okEvents); + await backfillRequiredFields(ctx, okEvents); assertEquals(ctx.payload.temperature, 0); }); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts index f8bb662f00..f2b68e00ce 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts +++ b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts @@ -23,7 +23,7 @@ test('captures a string system into a synthetic user/assistant pair and drops `s system: 'You are a pirate.', }); - await hoistUserSystemToMessages(ctx, {}, okEvents); + await hoistUserSystemToMessages(ctx, okEvents); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages, [ @@ -44,7 +44,7 @@ test('joins multi-block system into one synthetic turn with blank-line separator ], }); - await hoistUserSystemToMessages(ctx, {}, okEvents); + await hoistUserSystemToMessages(ctx, okEvents); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages[0], { role: 'user', content: [{ type: 'text', text: '[System Instructions]\nfirst rule\n\nsecond rule' }] }); @@ -57,7 +57,7 @@ test('drops system entirely when caller did not send one and leaves messages unt messages: [{ role: 'user', content: 'hi' }], }); - await hoistUserSystemToMessages(ctx, {}, okEvents); + await hoistUserSystemToMessages(ctx, okEvents); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages.length, 1); @@ -72,7 +72,7 @@ test('drops system but does not inject a synthetic turn when system text is empt system: [{ type: 'text', text: '' }], }); - await hoistUserSystemToMessages(ctx, {}, okEvents); + await hoistUserSystemToMessages(ctx, okEvents); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages.length, 1); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts index ca62714629..b564732440 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts +++ b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts @@ -23,7 +23,7 @@ test('drops a single billing block as system[0] with the pinned CLI version and messages: [{ role: 'user', content: 'hello world' }], }); - await injectBillingBlock(ctx, {}, okEvents); + await injectBillingBlock(ctx, okEvents); const system = ctx.payload.system; if (!Array.isArray(system)) throw new Error('expected system to be an array'); @@ -46,7 +46,7 @@ test('overwrites any pre-existing system array (hoist already ran)', async () => system: [{ type: 'text', text: 'stale leftover' } satisfies AnthropicMessagesTextBlock], }); - await injectBillingBlock(ctx, {}, okEvents); + await injectBillingBlock(ctx, okEvents); const system = ctx.payload.system; if (!Array.isArray(system)) throw new Error('expected system to be an array'); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts index 628f3290b7..706abcfab0 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts +++ b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts @@ -26,7 +26,7 @@ test('appends DEFAULT_TEMPLATE_BLOCK as system[2] with ephemeral cache_control i system: [billingBlock, IDENTITY_BLOCK], }); - await injectDefaultTemplate(ctx, {}, okEvents); + await injectDefaultTemplate(ctx, okEvents); assertEquals(ctx.payload.system, [billingBlock, IDENTITY_BLOCK, DEFAULT_TEMPLATE_BLOCK]); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); @@ -57,7 +57,7 @@ test('preserves ephemeral cache_control when caller already holds 3 breakpoints tools: [cachedTool], }); - await injectDefaultTemplate(ctx, {}, okEvents); + await injectDefaultTemplate(ctx, okEvents); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); assertEquals(ctx.payload.system.length, 4); @@ -94,7 +94,7 @@ test('demotes our cache_control when caller already holds 4 breakpoints (would b tools: [cachedTool], }); - await injectDefaultTemplate(ctx, {}, okEvents); + await injectDefaultTemplate(ctx, okEvents); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); assertEquals(ctx.payload.system.length, 4); @@ -124,7 +124,7 @@ test('demotes our cache_control when caller already exceeds the cap', async () = tools: [cachedTool], }); - await injectDefaultTemplate(ctx, {}, okEvents); + await injectDefaultTemplate(ctx, okEvents); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); const injected = ctx.payload.system.at(-1)!; diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts index f51a9a40aa..494c39567d 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts +++ b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts @@ -25,7 +25,7 @@ test('appends IDENTITY_BLOCK after an existing system[0] block', async () => { system: [billing], }); - await injectIdentityBlock(ctx, {}, okEvents); + await injectIdentityBlock(ctx, okEvents); assertEquals(ctx.payload.system, [billing, IDENTITY_BLOCK]); }); @@ -39,7 +39,7 @@ test('appends IDENTITY_BLOCK onto a one-block system array regardless of block c system: [existing], }); - await injectIdentityBlock(ctx, {}, okEvents); + await injectIdentityBlock(ctx, okEvents); assertEquals(ctx.payload.system, [existing, IDENTITY_BLOCK]); }); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts index 93a273dffa..e65ccdf8f5 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts +++ b/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts @@ -26,7 +26,7 @@ test('fills metadata.user_id with the new JSON shape when absent', async () => { messages: [{ role: 'user', content: 'hello' }], }); - await synthesizeMetadataUserId(ctx, {}, okEvents); + await synthesizeMetadataUserId(ctx, okEvents); const userId = ctx.payload.metadata?.user_id; if (typeof userId !== 'string') throw new Error('expected user_id to be a string'); @@ -43,8 +43,8 @@ test('fills metadata.user_id with the new JSON shape when absent', async () => { test('device_id is stable per upstream id', async () => { const a = invocation({ model: 'claude-sonnet-4-5-20250929', max_tokens: 1, messages: [{ role: 'user', content: 'a' }] }); const b = invocation({ model: 'claude-sonnet-4-5-20250929', max_tokens: 1, messages: [{ role: 'user', content: 'b' }] }); - await synthesizeMetadataUserId(a, {}, okEvents); - await synthesizeMetadataUserId(b, {}, okEvents); + await synthesizeMetadataUserId(a, okEvents); + await synthesizeMetadataUserId(b, okEvents); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; assertEquals(ad.deviceId, bd.deviceId); @@ -53,8 +53,8 @@ test('device_id is stable per upstream id', async () => { test('device_id differs across upstreams', async () => { const a = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'x' }] }, 'up_a'); const b = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'x' }] }, 'up_b'); - await synthesizeMetadataUserId(a, {}, okEvents); - await synthesizeMetadataUserId(b, {}, okEvents); + await synthesizeMetadataUserId(a, okEvents); + await synthesizeMetadataUserId(b, okEvents); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; if (ad.deviceId === bd.deviceId) throw new Error('expected different device ids per upstream'); @@ -63,8 +63,8 @@ test('device_id differs across upstreams', async () => { test('session_id is stable for same upstream + same first-user prefix', async () => { const a = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'prefix' }, { role: 'assistant', content: 'reply1' }] }); const b = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'prefix' }, { role: 'assistant', content: 'reply2' }] }); - await synthesizeMetadataUserId(a, {}, okEvents); - await synthesizeMetadataUserId(b, {}, okEvents); + await synthesizeMetadataUserId(a, okEvents); + await synthesizeMetadataUserId(b, okEvents); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; assertEquals(ad.sessionId, bd.sessionId); @@ -78,7 +78,7 @@ test('preserves a caller-supplied user_id verbatim', async () => { messages: [{ role: 'user', content: 'x' }], metadata: { user_id: explicit }, }); - await synthesizeMetadataUserId(ctx, {}, okEvents); + await synthesizeMetadataUserId(ctx, okEvents); assertEquals(ctx.payload.metadata?.user_id, explicit); }); @@ -105,8 +105,8 @@ test('session_id differs when system prompt is shared but user message differs ( // Drive the same step pair the production chain does: synthesize first, // then hoist. Synthesize sees the operator's real first user message; // hoist runs after and rewrites `messages` for the wire shape. - await synthesizeMetadataUserId(a, {}, () => hoistUserSystemToMessages(a, {}, okEvents)); - await synthesizeMetadataUserId(b, {}, () => hoistUserSystemToMessages(b, {}, okEvents)); + await synthesizeMetadataUserId(a, () => hoistUserSystemToMessages(a, okEvents)); + await synthesizeMetadataUserId(b, () => hoistUserSystemToMessages(b, okEvents)); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts index 63231fbccb..1bd7cf3fea 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts @@ -20,7 +20,6 @@ import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS } from '@floway-dev/protocols/an // see a fully-formed payload. Caller-supplied values are never overwritten. export const backfillRequiredFields = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const next = { ...ctx.payload }; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts index da8164a499..c9bc0fdf58 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts @@ -29,7 +29,6 @@ const SYNTHETIC_ACK = 'Understood. I will follow these instructions.'; // - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/gateway_service.go#L4480-L4486 export const hoistUserSystemToMessages = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const system: string | AnthropicMessagesTextBlock[] | undefined = ctx.payload.system; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts index 5dc9da6a1f..fef4d69900 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts @@ -45,7 +45,7 @@ import type { ProviderStreamResult } from '@floway-dev/provider'; export type { AnthropicMessagesBoundaryCtx } from './types.ts'; -export const CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY: readonly Interceptor>[] = [ +export const CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY: readonly Interceptor>[] = [ backfillRequiredFields, synthesizeMetadataUserId, hoistUserSystemToMessages, diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts index 2b7d13e17e..3c6cca908b 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts @@ -18,7 +18,6 @@ import { CLAUDE_CLI_VERSION } from '../../headers.ts'; // value than what the request body settles to and break CC mimicry. export const injectBillingBlock = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const fingerprint = computeCcVersionFingerprint(CLAUDE_CLI_VERSION, ctx.payload); diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts index 5407d8319e..ee473e524e 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts @@ -59,7 +59,6 @@ const countCacheBreakpoints = (payload: AnthropicMessagesPayload): number => { // when this runs. export const injectDefaultTemplate = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { if (!Array.isArray(ctx.payload.system)) { diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts index a6a768933f..fafea88760 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts @@ -5,7 +5,6 @@ import type { AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic // system[1]; relies on injectBillingBlock having materialized payload.system as an array (see ./index.ts chain order). export const injectIdentityBlock = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const system = ctx.payload.system as AnthropicMessagesTextBlock[]; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts index c5c4df68d4..3352d64c10 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts @@ -30,7 +30,6 @@ import type { AnthropicMessagesMessage, AnthropicMessagesPayload } from '@floway export const synthesizeMetadataUserId = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const existing = ctx.payload.metadata?.user_id; diff --git a/packages/provider-claude-code/src/provider.ts b/packages/provider-claude-code/src/provider.ts index 4accf2d41d..808bbfd797 100644 --- a/packages/provider-claude-code/src/provider.ts +++ b/packages/provider-claude-code/src/provider.ts @@ -97,9 +97,8 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { if (looksShaped) return await terminal(); - return await runInterceptors>( + return await runInterceptors>( ctx, - {}, CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY, terminal, ); diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts b/packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts index 62973a9ceb..011d0aeae0 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts +++ b/packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts @@ -15,7 +15,7 @@ import { assertEquals, readJsonRequest } from '@floway-dev/test-utils'; // carry generate-only fields (tools/reasoning/temperature/...) — the // per-action narrowing through `toCompactPayloadShape` is what closes that // gap. -const pivotGenerateToCompact: Interceptor = async (ctx, _env, run) => { +const pivotGenerateToCompact: Interceptor = async (ctx, run) => { ctx.action = 'compact'; return await run(); }; diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts b/packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts index d1aa644ab4..03df7358f5 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts +++ b/packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts @@ -6,8 +6,6 @@ import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from import type { ProviderStreamResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test', headers: new Headers() }); @@ -21,7 +19,7 @@ const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBo test('injects the default when instructions is absent', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await injectDefaultInstructions(ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -29,7 +27,7 @@ test('injects the default when instructions is absent', async () => { test('injects the default when instructions is an empty string', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: '' }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await injectDefaultInstructions(ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -37,7 +35,7 @@ test('injects the default when instructions is an empty string', async () => { test('injects the default when instructions is null', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: null }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await injectDefaultInstructions(ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -45,7 +43,7 @@ test('injects the default when instructions is null', async () => { test('preserves a caller-supplied instructions string', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: 'You are a pirate.' }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await injectDefaultInstructions(ctx, okEvents); assertEquals(ctx.payload.instructions, 'You are a pirate.'); }); @@ -64,7 +62,7 @@ test.each([ instructions: value as unknown as string, }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await injectDefaultInstructions(ctx, okEvents); assertEquals(ctx.payload.instructions, value); }, @@ -80,7 +78,7 @@ test('injects the default and preserves input items it does not own', async () = ], }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await injectDefaultInstructions(ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); assertEquals(ctx.payload.input, [ diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts b/packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts index 87ff5c15ea..43d2c72fb7 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts +++ b/packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts @@ -6,8 +6,6 @@ import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from import type { ProviderStreamResult } from '@floway-dev/provider'; import { assertEquals, assertFalse, stubProviderModel } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test', headers: new Headers() }); @@ -42,7 +40,7 @@ test('drops every field Codex rejects with Unsupported parameter', async () => { stream_options: { include_usage: true }, } as unknown as CanonicalOpenAIResponsesPayload); - await stripUnsupportedFields(ctx, stubRequest, okEvents); + await stripUnsupportedFields(ctx, okEvents); assertFalse('max_output_tokens' in ctx.payload); assertFalse('temperature' in ctx.payload); @@ -67,7 +65,7 @@ test('leaves supported fields intact', async () => { temperature: 0.7, }); - await stripUnsupportedFields(ctx, stubRequest, okEvents); + await stripUnsupportedFields(ctx, okEvents); assertEquals(ctx.payload.model, 'gpt-test'); assertEquals(ctx.payload.input, [{ type: 'message', role: 'user', content: 'hello' }]); @@ -82,7 +80,7 @@ test('payload without any unsupported fields is preserved as-is', async () => { const payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }; const ctx = invocation(payload); - await stripUnsupportedFields(ctx, stubRequest, okEvents); + await stripUnsupportedFields(ctx, okEvents); assertEquals(ctx.payload, { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); }); diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index 14c21eb9a5..34020a634d 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -344,7 +344,7 @@ describe('createCodexProvider', () => { const model = stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} } }); // @ts-expect-error: each method has a different body type; we only assert // the synthetic 405 envelope is what comes back. - const result = await instance.instance[method](model, {}, undefined, noopUpstreamCallOptions()) as { response: Response }; + const result = await instance.instance[method](model, undefined, noopUpstreamCallOptions()) as { response: Response }; expect(result.response.status).toBe(405); const body = await result.response.json() as { error: { type: string; message: string } }; expect(body.error.type).toBe('method_not_allowed'); diff --git a/packages/provider-codex/src/interceptors/openai-responses/index.ts b/packages/provider-codex/src/interceptors/openai-responses/index.ts index 873e279d15..379c26f386 100644 --- a/packages/provider-codex/src/interceptors/openai-responses/index.ts +++ b/packages/provider-codex/src/interceptors/openai-responses/index.ts @@ -15,7 +15,7 @@ import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; // terminal — the streaming `generate` + non-streaming `compact` dispatch — // returns its `ProviderOpenAIResponsesResult` directly without any per-frame // lift/lower step. -export const CODEX_OPENAI_RESPONSES_BOUNDARY: readonly Interceptor[] = [ +export const CODEX_OPENAI_RESPONSES_BOUNDARY: readonly Interceptor[] = [ injectDefaultInstructions, stripUnsupportedFields, ]; diff --git a/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts b/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts index eaa53fe37c..999ef066e7 100644 --- a/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts +++ b/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts @@ -6,7 +6,6 @@ import type { OpenAIResponsesBoundaryCtx } from './types.ts'; // https://github.com/im4codes/imcodes/blob/5f769d933dfd679e3a4d670183b0384a1baf62cd/src/agent/providers/codex-sdk.ts#L560-L579 export const injectDefaultInstructions = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const instructions = ctx.payload.instructions; diff --git a/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts b/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts index a6d8e863bb..497930df35 100644 --- a/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts +++ b/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts @@ -22,7 +22,6 @@ const CODEX_UNSUPPORTED_BODY_FIELDS = [ export const stripUnsupportedFields = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const next: Record = { ...(ctx.payload as unknown as Record) }; diff --git a/packages/provider-codex/src/provider.ts b/packages/provider-codex/src/provider.ts index df3f1d0279..9ac46c0caf 100644 --- a/packages/provider-codex/src/provider.ts +++ b/packages/provider-codex/src/provider.ts @@ -136,8 +136,8 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { model, action, }; - return await runInterceptors( - ctx, {}, CODEX_OPENAI_RESPONSES_BOUNDARY, async () => { + return await runInterceptors( + ctx, CODEX_OPENAI_RESPONSES_BOUNDARY, async () => { const { account } = await readActiveAccount(); const { model: _ignored, ...wireBody } = ctx.payload; const backendCallBase = { upstreamId: record.id, account, model, headers: ctx.headers, signal, effects, call: opts }; diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts index 99da6bf30a..ea5ba8fee7 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -32,7 +30,7 @@ test('ports top-level cache_control onto the last cacheable block and drops the ], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await withTopLevelCacheControlApplied(ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[2].content, [{ type: 'text', text: 'second', cache_control: { type: 'ephemeral' } }]); @@ -48,7 +46,7 @@ test('respects an explicit cache_control on the last cacheable block', async () messages: [{ role: 'user', content: [{ type: 'text', text: 'hi', cache_control: explicit }] }], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await withTopLevelCacheControlApplied(ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'hi', cache_control: explicit }]); @@ -62,7 +60,7 @@ test('promotes string message content so the auto-marker has a block to land on' messages: [{ role: 'user', content: 'plain' }], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await withTopLevelCacheControlApplied(ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'plain', cache_control: { type: 'ephemeral' } }]); @@ -79,7 +77,7 @@ test('walks past trailing non-cacheable blocks to find the last cacheable one', ], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await withTopLevelCacheControlApplied(ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[1].content, [{ type: 'thinking', thinking: 'pondering' }]); @@ -94,7 +92,7 @@ test('drops top-level cache_control when no cacheable block exists', async () => messages: [{ role: 'assistant', content: [{ type: 'thinking', thinking: 'no text yet' }] }], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await withTopLevelCacheControlApplied(ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'thinking', thinking: 'no text yet' }]); @@ -107,7 +105,7 @@ test('no-op when payload has no top-level cache_control', async () => { messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await withTopLevelCacheControlApplied(ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'hi' }]); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts index dc3742bb29..aa2daf658a 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts @@ -10,8 +10,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -41,9 +39,8 @@ test('Claude Code SDK compact request: Claude-agent overrides compact intent, bo messages: [{ role: 'user', content: COMPACT_LAST_MESSAGE_TEXT }], }); - await runInterceptors>>( + await runInterceptors>>( ctx, - stubRequest, COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, okEvents, ); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts index 8835ddccd8..808f84d0eb 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts @@ -9,8 +9,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -59,7 +57,7 @@ test('compresses a top-level image block to WebP', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); const block = (ctx.payload.messages[0].content as Array<{ type: string; source?: { media_type: string; data: string } }>)[1]; assertEquals(block.source?.media_type, 'image/webp'); @@ -110,7 +108,7 @@ test('compresses an image nested inside tool_result content', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); const rewrittenContent = ctx.payload.messages[0].content; if (!Array.isArray(rewrittenContent)) throw new Error('expected rewritten user content'); @@ -144,7 +142,7 @@ test('leaves image-free payloads untouched and does not invoke the processor', a }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(inputs.length, 0); assert(ctx.payload === payload); @@ -174,7 +172,7 @@ test('compresses each unique inline image only once when the same base64 data re ], }); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(inputs.length, 2); }); @@ -214,7 +212,7 @@ const capTargetFor = async (upstreamModelId: string, source: ImageDimensions): P }, ], }, upstreamModelId); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); return targets[0]; }; diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts index 1d03f5c413..65f57302d4 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts @@ -28,8 +28,6 @@ const makeCtx = (speed?: unknown): AnthropicMessagesBoundaryCtx => ({ model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), }); -const stubRequest = {}; - const messageStart = (usage: AnthropicMessagesUsage): ProtocolFrame => eventFrame({ type: 'message_start', message: { @@ -63,7 +61,7 @@ const streamResult = (frames: ProtocolFrame[]): Ex test('withSpeedFast strips speed=fast from the outbound payload', async () => { const ctx = makeCtx('fast'); - await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult([]))); + await withSpeedFast(ctx, () => Promise.resolve(streamResult([]))); assertEquals('speed' in ctx.payload, false); }); @@ -71,7 +69,7 @@ test('withSpeedFast strips speed=fast from the outbound payload', async () => { test('withSpeedFast strips speed=standard (semantically equivalent to omitted)', async () => { const ctx = makeCtx('standard'); - await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult([]))); + await withSpeedFast(ctx, () => Promise.resolve(streamResult([]))); assertEquals('speed' in ctx.payload, false); }); @@ -83,7 +81,7 @@ test('withSpeedFast leaves unknown speed values untouched so the upstream reject // error. We never invent a fall-through here. const ctx = makeCtx('priority'); - await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult([]))); + await withSpeedFast(ctx, () => Promise.resolve(streamResult([]))); assertEquals(ctx.payload.speed, 'priority'); }); @@ -91,7 +89,7 @@ test('withSpeedFast leaves unknown speed values untouched so the upstream reject test('withSpeedFast stamps usage.speed=fast on message_start when fast was requested', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, stubRequest, () => + const result = await withSpeedFast(ctx, () => Promise.resolve(streamResult([messageStart(baseUsage), messageStop(), doneFrame()]))); assertEquals(result.type, 'events'); @@ -107,7 +105,7 @@ test('withSpeedFast stamps usage.speed=fast on message_start when fast was reque test('withSpeedFast stamps usage.speed=fast on every message_delta carrying usage', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, stubRequest, () => + const result = await withSpeedFast(ctx, () => Promise.resolve(streamResult([messageStart(baseUsage), messageDelta(7), messageStop()]))); assertEquals(result.type, 'events'); @@ -128,7 +126,7 @@ test('withSpeedFast leaves the stream untouched when speed is absent', async () const ctx = makeCtx(); const frames = [messageStart(baseUsage), messageDelta(3), messageStop()]; - const result = await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult(frames))); + const result = await withSpeedFast(ctx, () => Promise.resolve(streamResult(frames))); assertEquals(result.type, 'events'); if (result.type !== 'events') throw new Error('expected events'); @@ -140,7 +138,7 @@ test('withSpeedFast leaves the stream untouched when speed=standard (no fast int const ctx = makeCtx('standard'); const frames = [messageStart(baseUsage), messageStop()]; - const result = await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult(frames))); + const result = await withSpeedFast(ctx, () => Promise.resolve(streamResult(frames))); assertEquals(result.type, 'events'); if (result.type !== 'events') throw new Error('expected events'); @@ -150,7 +148,7 @@ test('withSpeedFast leaves the stream untouched when speed=standard (no fast int test('withSpeedFast surfaces non-events results (api-error / internal-error) verbatim', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, stubRequest, () => + const result = await withSpeedFast(ctx, () => Promise.resolve({ type: 'internal-error', status: 502, diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts index 560d6bffdf..9a54c4fbb6 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts @@ -7,7 +7,6 @@ import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { eventResult, type ExecuteResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -34,7 +33,7 @@ test('keeps only supported caller beta values in first-seen order', async () => 'advanced-tool-use-2025-11-20', 'context-1m-2025-08-07', ]); - await withAnthropicBetaNormalized(ctx, stubRequest, okEvents); + await withAnthropicBetaNormalized(ctx, okEvents); assertEquals(ctx.anthropicBeta, ['advanced-tool-use-2025-11-20']); }); @@ -42,15 +41,15 @@ test('synthesizes interleaved thinking only when the caller supplied no beta int const payload = { ...baseBody, thinking: { type: 'enabled' as const, budget_tokens: 1024 } }; const silent = invocation(payload); const explicit = invocation(payload, ['context-1m-2025-08-07']); - await withAnthropicBetaNormalized(silent, stubRequest, okEvents); - await withAnthropicBetaNormalized(explicit, stubRequest, okEvents); + await withAnthropicBetaNormalized(silent, okEvents); + await withAnthropicBetaNormalized(explicit, okEvents); assertEquals(silent.anthropicBeta, ['interleaved-thinking-2025-05-14']); assertEquals(explicit.anthropicBeta, []); }); test('does not synthesize interleaved thinking for adaptive thinking', async () => { const ctx = invocation({ ...baseBody, thinking: { type: 'adaptive', budget_tokens: 1024 } }); - await withAnthropicBetaNormalized(ctx, stubRequest, okEvents); + await withAnthropicBetaNormalized(ctx, okEvents); assertEquals(ctx.anthropicBeta, []); }); @@ -59,7 +58,7 @@ test('pairs context management with its required beta token', async () => { { ...baseBody, context_management: { edits: [] } }, ['interleaved-thinking-2025-05-14'], ); - await withAnthropicBetaNormalized(ctx, stubRequest, okEvents); + await withAnthropicBetaNormalized(ctx, okEvents); assertEquals(ctx.anthropicBeta, [ 'interleaved-thinking-2025-05-14', 'context-management-2025-06-27', diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts index 0af6c7c173..1d64787749 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts @@ -31,8 +31,6 @@ const makeCtx = ( model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), }); -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); test('resolveAnthropicMessagesDownstreamThinkingDisplay exposes 4.7+ omitted by default and older Claude as summarized', () => { @@ -85,7 +83,7 @@ test('resolveAnthropicMessagesDownstreamThinkingDisplay ignores unknown explicit test('withThinkingDisplayPromoted sends summarized upstream when thinking display is omitted', async () => { const ctx = makeCtx({ type: 'adaptive' }); - await withThinkingDisplayPromoted(ctx, stubRequest, () => + await withThinkingDisplayPromoted(ctx, () => Promise.resolve({ type: 'internal-error', status: 418, @@ -105,8 +103,8 @@ test('withThinkingDisplayPromoted overrides omitted but preserves full', async ( const omittedCtx = makeCtx({ type: 'adaptive', display: 'omitted' }); const fullCtx = makeCtx({ type: 'adaptive', display: 'full' }); - await withThinkingDisplayPromoted(omittedCtx, stubRequest, okEvents); - await withThinkingDisplayPromoted(fullCtx, stubRequest, okEvents); + await withThinkingDisplayPromoted(omittedCtx, okEvents); + await withThinkingDisplayPromoted(fullCtx, okEvents); assertEquals(omittedCtx.payload.thinking?.display, 'summarized'); assertEquals(fullCtx.payload.thinking?.display, 'full'); @@ -116,8 +114,8 @@ test('withThinkingDisplayPromoted leaves disabled or absent thinking untouched', const disabledCtx = makeCtx({ type: 'disabled' }); const absentCtx = makeCtx(undefined); - await withThinkingDisplayPromoted(disabledCtx, stubRequest, okEvents); - await withThinkingDisplayPromoted(absentCtx, stubRequest, okEvents); + await withThinkingDisplayPromoted(disabledCtx, okEvents); + await withThinkingDisplayPromoted(absentCtx, okEvents); assertEquals(disabledCtx.payload.thinking, { type: 'disabled' }); assertEquals(absentCtx.payload.thinking, undefined); @@ -127,7 +125,7 @@ test('withThinkingDisplayPromoted leaves unknown display values for upstream val const ctx = makeCtx({ type: 'adaptive' }); (ctx.payload.thinking as { display?: unknown }).display = 'omit'; - await withThinkingDisplayPromoted(ctx, stubRequest, okEvents); + await withThinkingDisplayPromoted(ctx, okEvents); assertEquals((ctx.payload.thinking as { display?: unknown }).display, 'omit'); }); @@ -135,7 +133,7 @@ test('withThinkingDisplayPromoted leaves unknown display values for upstream val test('withThinkingDisplayPromoted simulates omitted display on protocol events', async () => { const ctx = makeCtx({ type: 'adaptive' }); - const result = await withThinkingDisplayPromoted(ctx, stubRequest, () => + const result = await withThinkingDisplayPromoted(ctx, () => Promise.resolve( eventResult( (async function* (): AsyncGenerator> { diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts index dcd035ba0a..c1a6bb751c 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts @@ -9,8 +9,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -31,7 +29,7 @@ const basePayload = (userId: string | undefined): AnthropicMessagesPayload => ({ test('Claude agent headers set for the legacy fingerprint with both halves', async () => { const ctx = invocation(basePayload('user_acct-1_account__session_sess-1')); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await withClaudeAgentHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'messages-proxy'); assertEquals(ctx.headers.get('openai-intent'), 'messages-proxy'); @@ -44,7 +42,7 @@ test('Claude agent headers set for the legacy fingerprint with both halves', asy test('Claude agent headers set for the JSON fingerprint with device_id + session_id', async () => { const ctx = invocation(basePayload(JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }))); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await withClaudeAgentHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('user-agent'), CLAUDE_AGENT_USER_AGENT); assertEquals(ctx.headers.get('copilot-integration-id'), ''); @@ -53,7 +51,7 @@ test('Claude agent headers set for the JSON fingerprint with device_id + session test('Claude agent headers absent when session_id is missing', async () => { const ctx = invocation(basePayload(JSON.stringify({ device_id: 'dev-1' }))); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await withClaudeAgentHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -62,7 +60,7 @@ test('Claude agent headers absent when session_id is missing', async () => { test('Claude agent headers absent when safety identifier is missing', async () => { const ctx = invocation(basePayload(JSON.stringify({ session_id: 'sess-only' }))); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await withClaudeAgentHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -71,7 +69,7 @@ test('Claude agent headers absent when safety identifier is missing', async () = test('Claude agent headers absent when metadata is not provided', async () => { const ctx = invocation(basePayload(undefined)); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await withClaudeAgentHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -80,7 +78,7 @@ test('Claude agent headers absent when metadata is not provided', async () => { test('Claude agent headers skipped on claude-opus-4-8 even with full fingerprint', async () => { const ctx = invocation({ ...basePayload(JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' })), model: 'claude-opus-4-8' }); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await withClaudeAgentHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('openai-intent'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts index 308b0d01e9..3441fb11e9 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -32,7 +30,7 @@ test('Compact headers set when the last user message carries all three markers', messages: [{ role: 'user', content: COMPACT_LAST_MESSAGE_TEXT }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); @@ -54,7 +52,7 @@ test('Compact headers set from a multi-block last message that joins to the full ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); }); @@ -67,7 +65,7 @@ test('Compact headers set when the system prompt starts with a compact summariza messages: [{ role: 'user', content: 'go ahead' }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); @@ -84,7 +82,7 @@ test('Compact headers set when an array system prompt contains a compact prefix messages: [{ role: 'user', content: 'hi' }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); }); @@ -96,7 +94,7 @@ test('Compact headers absent when only the text-only guard is present (other mar messages: [{ role: 'user', content: 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.' }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -109,7 +107,7 @@ test('Compact headers absent for an ordinary user turn', async () => { messages: [{ role: 'user', content: 'hello there' }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -130,7 +128,7 @@ test('Compact headers absent when the last message is assistant-role with compac ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -152,7 +150,7 @@ test('Compact headers absent when the last assistant message is a multi-block co ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -171,7 +169,7 @@ test('Auto-continue absent when the assistant role carries the resume prompt ver ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -189,7 +187,7 @@ test('Auto-continue marks Claude Code resume prompts with x-initiator: agent onl ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -207,7 +205,7 @@ test('Auto-continue marks OpenCode primary continuation prompts with x-initiator ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -225,7 +223,7 @@ test('Auto-continue marks OpenCode media-eviction continuation prompts with x-in ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await withCompactHeadersSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts index 86e88414cd..c36fdebbf7 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -27,7 +25,7 @@ test('Anthropic Messages initiator is user when the last message is a plain user messages: [{ role: 'user', content: 'hello' }], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -47,7 +45,7 @@ test('Anthropic Messages initiator is user when the last user turn mixes text an ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -66,7 +64,7 @@ test('Anthropic Messages initiator is agent when the last user turn is entirely ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -81,7 +79,7 @@ test('Anthropic Messages initiator is agent when the final message is from the a ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts index c042cfb3b0..d160861b28 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts @@ -9,8 +9,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -39,7 +37,7 @@ const SESS_ALONE_UUID = '19be0c9e-9a65-43eb-be80-efb2770d4510'; test('Interaction-id forwarded as a SHA-256 UUID for the legacy fingerprint with both halves', async () => { const ctx = invocation(payloadWith('user_acct-1_account__session_sess-legacy')); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await withInteractionIdHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_LEGACY_UUID); }); @@ -47,7 +45,7 @@ test('Interaction-id forwarded as a SHA-256 UUID for the legacy fingerprint with test('Interaction-id forwarded as a SHA-256 UUID for the JSON fingerprint carrying session_id', async () => { const ctx = invocation(payloadWith(JSON.stringify({ device_id: 'dev', session_id: 'sess-json' }))); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await withInteractionIdHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_JSON_UUID); }); @@ -57,7 +55,7 @@ test('Interaction-id forwarded as a SHA-256 UUID even when only the sessionId ha // device_id / account_uuid; we still want trace correlation in that case. const ctx = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await withInteractionIdHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_ALONE_UUID); }); @@ -66,8 +64,8 @@ test('Interaction-id hashing is deterministic across repeated invocations', asyn const first = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); const second = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); - await withInteractionIdHeaderSet(first, stubRequest, okEvents); - await withInteractionIdHeaderSet(second, stubRequest, okEvents); + await withInteractionIdHeaderSet(first, okEvents); + await withInteractionIdHeaderSet(second, okEvents); assertEquals(first.headers.get('x-interaction-id'), second.headers.get('x-interaction-id')); }); @@ -75,7 +73,7 @@ test('Interaction-id hashing is deterministic across repeated invocations', asyn test('Interaction-id has RFC UUIDv4 version and variant bits', async () => { const ctx = invocation(payloadWith(JSON.stringify({ session_id: 'sess-shape-check' }))); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await withInteractionIdHeaderSet(ctx, okEvents); const value = ctx.headers.get('x-interaction-id'); assert(value !== null && validate(value)); @@ -87,7 +85,7 @@ test('Interaction-id has RFC UUIDv4 version and variant bits', async () => { test('Interaction-id absent when metadata.user_id is missing', async () => { const ctx = invocation(payloadWith(undefined)); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await withInteractionIdHeaderSet(ctx, okEvents); assertEquals(ctx.headers.has('x-interaction-id'), false); }); @@ -95,7 +93,7 @@ test('Interaction-id absent when metadata.user_id is missing', async () => { test('Interaction-id absent when metadata.user_id carries no session marker', async () => { const ctx = invocation(payloadWith('user_acct-1_account')); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await withInteractionIdHeaderSet(ctx, okEvents); assertEquals(ctx.headers.has('x-interaction-id'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts index 6e6677b542..3a24e133c5 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -35,7 +33,7 @@ test('Anthropic Messages vision header set when a top-level image block is prese ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await withVisionHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -60,7 +58,7 @@ test('Anthropic Messages vision header set when an image is nested inside tool_r ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await withVisionHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -83,7 +81,7 @@ test('Anthropic Messages vision header absent when no image is present', async ( ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await withVisionHeaderSet(ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts index 68e7f027e8..88d1082462 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -52,7 +50,7 @@ test('strips scope and ttl from system, tools, and message content blocks while ], }); - await withCacheControlExtensionsStripped(ctx, stubRequest, okEvents); + await withCacheControlExtensionsStripped(ctx, okEvents); assertEquals(ctx.payload.system, [{ type: 'text', text: 'sys', cache_control: { type: 'ephemeral' } }]); assertEquals(ctx.payload.tools, [{ name: 't', input_schema: { type: 'object' }, cache_control: { type: 'ephemeral' } }]); @@ -74,7 +72,7 @@ test('deletes cache_control entirely if no recognised field survives the strip', messages: [{ role: 'user', content: 'hi' }], }); - await withCacheControlExtensionsStripped(ctx, stubRequest, okEvents); + await withCacheControlExtensionsStripped(ctx, okEvents); assertEquals(ctx.payload.system, [{ type: 'text', text: 'sys' }]); }); @@ -88,7 +86,7 @@ test('no-op when no cache_control is present anywhere', async () => { tools: [{ name: 't', input_schema: { type: 'object' } }], }); - await withCacheControlExtensionsStripped(ctx, stubRequest, okEvents); + await withCacheControlExtensionsStripped(ctx, okEvents); assertEquals(ctx.payload.system, 'plain system'); assertEquals(ctx.payload.tools, [{ name: 't', input_schema: { type: 'object' } }]); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts index 44ae3ee9de..ae8695d70f 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts @@ -15,8 +15,6 @@ const invocation = (): OpenAIChatCompletionsBoundaryCtx => ({ model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), }); -const stubRequest = {}; - const baseChunk = (overrides: Partial): OpenAIChatCompletionsStreamEvent => ({ id: 'chatcmpl_1', object: 'chat.completion.chunk', @@ -34,7 +32,7 @@ const collect = async (result: ExecuteResult[]): Promise[]> => { - const result = await withToolArgumentWhitespaceAborted(invocation(), stubRequest, () => + const result = await withToolArgumentWhitespaceAborted(invocation(), () => Promise.resolve( eventResult( (async function* () { diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts index bb51f13ce0..d0a02d4c89 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assert, assertEquals, assertFalse, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -32,7 +30,7 @@ test('OpenAI Chat Completions cache markers attach to first two systems and last { role: 'assistant', content: 'assistant 2' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await withCacheControlMarkersAttached(ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [0, 1, 4, 5]); for (const index of [0, 1, 4, 5]) { @@ -45,7 +43,7 @@ test('OpenAI Chat Completions cache markers handle a single system message', asy { role: 'system', content: 'only system' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await withCacheControlMarkersAttached(ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [0]); }); @@ -58,7 +56,7 @@ test('OpenAI Chat Completions cache markers attach to last two non-systems when { role: 'assistant', content: 'assistant 2' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await withCacheControlMarkersAttached(ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [2, 3]); }); @@ -69,7 +67,7 @@ test('OpenAI Chat Completions cache markers skip empty string content', async () { role: 'user', content: '' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await withCacheControlMarkersAttached(ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), []); }); @@ -80,7 +78,7 @@ test('OpenAI Chat Completions cache markers skip empty array content', async () { role: 'user', content: [] }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await withCacheControlMarkersAttached(ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), []); }); @@ -95,7 +93,7 @@ test('OpenAI Chat Completions cache markers only mark first two of five systems' { role: 'user', content: 'user 1' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await withCacheControlMarkersAttached(ctx, okEvents); // First two systems + last (and only) non-system. assertEquals(markedIndexes(ctx.payload.messages), [0, 1, 5]); @@ -111,7 +109,7 @@ test('OpenAI Chat Completions cache markers are independent object instances per { role: 'user', content: 'user 1' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await withCacheControlMarkersAttached(ctx, okEvents); const a = (ctx.payload.messages[0] as CopilotCacheableMessage).copilot_cache_control; const b = (ctx.payload.messages[1] as CopilotCacheableMessage).copilot_cache_control; diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts index 1da921fbfa..381ef108e8 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts @@ -9,8 +9,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -47,7 +45,7 @@ test('rewrites a base64 image_url data URL to a WebP data URL', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(imageUrl(ctx.payload), 'data:image/webp;base64,AQID'); assertEquals(imageUrl(payload), 'data:image/png;base64,AAAA'); @@ -79,7 +77,7 @@ test('leaves remote https image references untouched', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(imageUrl(ctx.payload), 'https://example.com/cat.png'); assert(ctx.payload === payload); @@ -114,7 +112,7 @@ test('compresses each unique inline image only once when the same data URL repea ], }); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(calls, 2); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts index 3ed8f66085..693d2445e9 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -25,7 +23,7 @@ test('OpenAI Chat Completions initiator is user when the last message is from th messages: [{ role: 'user', content: 'hello' }], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -39,7 +37,7 @@ test('OpenAI Chat Completions initiator is agent when the last message is an ass ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -58,7 +56,7 @@ test('OpenAI Chat Completions initiator is agent when the last message is a tool ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -78,7 +76,7 @@ test('OpenAI Chat Completions initiator follows the final user role for a lifted ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts index 63605f9452..4bf5f3bd9d 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -33,7 +31,7 @@ test('OpenAI Chat Completions vision header set when an image_url content part i ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await withVisionHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -47,7 +45,7 @@ test('OpenAI Chat Completions vision header absent when content is pure text', a ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await withVisionHeaderSet(ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts index 0436fecb30..4bd4e89279 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts @@ -28,8 +28,6 @@ const invocation = (): OpenAIResponsesBoundaryCtx => ({ action: 'generate', }); -const stubRequest = {}; - const argsDelta = (outputIndex: number, delta: string): OpenAIResponsesStreamEvent => ({ type: 'response.function_call_arguments.delta', @@ -46,7 +44,7 @@ const collect = async (result: ProviderOpenAIResponsesResult): Promise[]): Promise[]> => { - const result = await withToolArgumentWhitespaceAborted(invocation(), stubRequest, () => + const result = await withToolArgumentWhitespaceAborted(invocation(), () => Promise.resolve({ action: 'generate', ok: true, diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts index 0ed25c8408..542d87b503 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts @@ -14,7 +14,7 @@ import { assertEquals } from '@floway-dev/test-utils'; // observed wire request must be the streaming /responses shape // (stream:true, no `compaction_trigger`), and the typed result must // surface as the `action: 'generate'` variant. -const pivotCompactToGenerate: Interceptor = async (ctx, _env, run) => { +const pivotCompactToGenerate: Interceptor = async (ctx, run) => { ctx.action = 'generate'; return await run(); }; diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts index 33bec4785f..3ef33a8adb 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts @@ -9,8 +9,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -55,7 +53,7 @@ test.each(Object.entries(contentContainers))('compresses inline images in %s', a }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(imageUrlOf(ctx.payload.input[0]), 'data:image/webp;base64,AQID'); assertEquals(imageUrlOf(sourceItem), 'data:image/png;base64,AAAA'); @@ -81,7 +79,7 @@ test.each(Object.entries(contentContainers))('leaves remote images in %s untouch }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(imageUrlOf(ctx.payload.input[0]), 'https://example.com/cat.png'); assert(ctx.payload === payload); @@ -127,7 +125,7 @@ test('compresses each unique inline image only once when the same data URL appea ], }); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); // Two unique data URLs across five targets → exactly two compress calls. assertEquals(calls, 2); @@ -151,8 +149,8 @@ test('reuses its compressed image when the same attempt payload is retried', asy }], }); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(calls, 1); assertEquals(imageUrlOf(ctx.payload.input[0]), 'data:image/webp;base64,AQID'); @@ -160,7 +158,7 @@ test('reuses its compressed image when the same attempt payload is retried', asy const item = ctx.payload.input[0]; if (item.type !== 'message' || !Array.isArray(item.content) || item.content[0]?.type !== 'input_image') throw new Error('expected image content'); item.content[0].image_url = 'data:image/png;base64,BBBB'; - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await withInlineImagesCompressed(ctx, okEvents); assertEquals(calls, 2); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts index d50a82f583..3f4538e1ec 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts @@ -36,7 +36,7 @@ test('fills Codex OpenAI Responses Lite namespace descriptions before Copilot di }], }); - await withEmptyNamespaceDescriptionsFilled(ctx, {}, async () => {}); + await withEmptyNamespaceDescriptionsFilled(ctx, async () => {}); const [item] = ctx.payload.input; if (item?.type !== 'additional_tools') throw new Error('expected additional_tools input'); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts index 6e69cfca35..8c6dcee197 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -23,7 +21,7 @@ const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBo test('forces store:false when the caller requested store:true', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], store: true }); - await withStoreForcedFalse(ctx, stubRequest, okEvents); + await withStoreForcedFalse(ctx, okEvents); assertEquals(ctx.payload.store, false); }); @@ -31,7 +29,7 @@ test('forces store:false when the caller requested store:true', async () => { test('sets store:false when the caller omitted store', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); - await withStoreForcedFalse(ctx, stubRequest, okEvents); + await withStoreForcedFalse(ctx, okEvents); assertEquals(ctx.payload.store, false); }); @@ -39,7 +37,7 @@ test('sets store:false when the caller omitted store', async () => { test('leaves an explicit store:false untouched', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], store: false }); - await withStoreForcedFalse(ctx, stubRequest, okEvents); + await withStoreForcedFalse(ctx, okEvents); assertEquals(ctx.payload.store, false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts index 382895cbeb..daca6c17e2 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts @@ -53,7 +53,7 @@ const runStream = async ( const iterable = Symbol.asyncIterator in frames ? frames as AsyncIterable> : (async function* () { yield* frames as ProtocolFrame[]; })(); - const result = await withCopilotOpenAIResponsesItemIdMembrane(ctx, {}, () => Promise.resolve({ + const result = await withCopilotOpenAIResponsesItemIdMembrane(ctx, () => Promise.resolve({ action: 'generate', ok: true, events: iterable, @@ -404,7 +404,7 @@ test('restores owned blob ids for Copilot input and leaves foreign items unchang ]; const ctx = invocation(input); let wireInput: OpenAIResponsesInputItem[] | undefined; - await withCopilotOpenAIResponsesItemIdMembrane(ctx, {}, () => { + await withCopilotOpenAIResponsesItemIdMembrane(ctx, () => { wireInput = structuredClone(ctx.payload.input); return Promise.resolve({ action: 'generate', @@ -445,7 +445,7 @@ test('rejects conflicting ids carried by one input item', async () => { ], }]); - await expect(withCopilotOpenAIResponsesItemIdMembrane(ctx, {}, () => { + await expect(withCopilotOpenAIResponsesItemIdMembrane(ctx, () => { throw new Error('must not reach upstream'); })).rejects.toThrow(/conflicting upstream ids/); }); @@ -455,7 +455,7 @@ test('normalizes the generated compaction item without touching retained compact { type: 'message', id: 'msg_retained', status: 'completed', role: 'assistant', content: [] }, { type: 'compaction', id: 'cmp_raw', encrypted_content: 'compact state' }, ]); - const result = await withCopilotOpenAIResponsesItemIdMembrane(invocation(), {}, () => Promise.resolve({ + const result = await withCopilotOpenAIResponsesItemIdMembrane(invocation(), () => Promise.resolve({ action: 'compact', ok: true, result: compactResult, diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts index 4fa9098ad1..8af3e3e8d1 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -32,7 +30,7 @@ test.each(['user', 'system', 'developer'] as const)('OpenAI Responses initiator ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -40,7 +38,7 @@ test.each(['user', 'system', 'developer'] as const)('OpenAI Responses initiator test('OpenAI Responses initiator is user when input is an empty array', async () => { const ctx = invocation({ model: 'gpt-test', input: [] }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -55,7 +53,7 @@ test('OpenAI Responses initiator is user for the role-bearing additional_tools i }], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -84,7 +82,7 @@ test('OpenAI Responses initiator is agent when the last input item is a function ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -112,7 +110,7 @@ test('OpenAI Responses initiator is agent when the last input item is a custom_t ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -130,7 +128,7 @@ test('OpenAI Responses initiator is agent when the last canonical item is reason ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -152,7 +150,7 @@ test('OpenAI Responses initiator is agent when the last input item is an assista ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await withInitiatorHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts b/packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts index 54b1b1041e..66d10d45c7 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts @@ -8,8 +8,6 @@ import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -35,7 +33,7 @@ test.each(Object.entries(contentContainers))('OpenAI Responses vision header det ])], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await withVisionHeaderSet(ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -46,7 +44,7 @@ test.each(Object.entries(contentContainers))('OpenAI Responses vision header ign input: [wrap([{ type: 'input_text', text: 'plain text only' }])], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await withVisionHeaderSet(ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts index cc4dbb9624..23051edc04 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts @@ -50,7 +50,7 @@ const markerSite = (messages: AnthropicMessagesPayload['messages']): { readonly return undefined; }; -export const withTopLevelCacheControlApplied: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { +export const withTopLevelCacheControlApplied: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { const payload = ctx.payload as typeof ctx.payload & { cache_control?: { type: 'ephemeral' } }; const topLevel = payload.cache_control; if (topLevel === undefined) return await run(); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts index e8874fb1ca..9e56a357ff 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts @@ -105,7 +105,6 @@ const compressInlineImages = async (ctx: AnthropicMessagesBoundaryCtx): Promise< // payload the chat path sends. export const withInlineImagesCompressed = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { // Finish this nested activation before starting the upstream call. Its diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts index 29f3175c30..e4d840d865 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts @@ -27,7 +27,7 @@ import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; * - https://docs.claude.com/en/build-with-claude/fast-mode * - https://docs.claude.com/en/api/service-tiers */ -export const withSpeedFast: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { +export const withSpeedFast: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { const speed = ctx.payload.speed; const stampFast = speed === 'fast'; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts index 70d6986216..b7c88d187b 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts @@ -16,7 +16,6 @@ const CONTEXT_MANAGEMENT_BETA = 'context-management-2025-06-27'; // https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/extension/chatSessions/claude/node/claudeLanguageModelServer.ts#L413-L427 export const withAnthropicBetaNormalized = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const callerSuppliedBeta = ctx.anthropicBeta.length > 0; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts index d869a42fd8..bc406b93d1 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts @@ -91,7 +91,7 @@ const omitThinkingTextFromProtocolFrames = async function* (frames: AsyncIterabl * - https://github.com/anthropics/claude-code/issues/46987 * - https://github.com/anthropics/claude-code/issues/50477 */ -export const withThinkingDisplayPromoted: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { +export const withThinkingDisplayPromoted: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { const downstreamDisplay = resolveAnthropicMessagesDownstreamThinkingDisplay(ctx); const thinking = ctx.payload.thinking; const hasActiveThinking = !!thinking && thinking.type !== 'disabled'; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts index 62a20057c9..9b8914c8c2 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts @@ -17,7 +17,7 @@ const isContextWindowError = (text: string): boolean => text.includes('Request b * substring set here (`Request body is too large...`) is disjoint from * the OpenAI Responses/OpenAI Chat Completions shapes those pairs match on. */ -export const rewriteContextWindowError: CopilotAnthropicMessagesBoundaryInterceptor = async (_ctx, _env, run) => { +export const rewriteContextWindowError: CopilotAnthropicMessagesBoundaryInterceptor = async (_ctx, run) => { const result = await run(); if (result.type !== 'api-error' || result.source !== 'upstream') return result; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts index 036af530cb..edf4f32297 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts @@ -42,7 +42,7 @@ import { CLAUDE_AGENT_USER_AGENT } from '../../auth.ts'; */ const UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY = new Set(['claude-opus-4-8']); -export const withClaudeAgentHeadersSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { +export const withClaudeAgentHeadersSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { if (UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY.has(ctx.payload.model)) { return await run(); } diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts index 4ab55d9361..e9ec7876c3 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts @@ -105,7 +105,7 @@ const classifyCompact = (payload: { messages: AnthropicMessagesMessage[]; system return null; }; -export const withCompactHeadersSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { +export const withCompactHeadersSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { const kind = classifyCompact(ctx.payload); if (kind === 'compact-request') { ctx.headers.set('x-initiator', 'agent'); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts index e1aaff3a66..aae4ccf536 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts @@ -23,7 +23,6 @@ import type { AnthropicMessagesBoundaryCtx } from './types.ts'; */ export const withInitiatorHeaderSet = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const lastMessage = ctx.payload.messages[ctx.payload.messages.length - 1]; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts index 0315638e69..2dfa0cfc08 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts @@ -24,7 +24,7 @@ const sessionUuid = async (input: string): Promise => { return v4({ random: new Uint8Array(await crypto.subtle.digest('SHA-256', data)) }); }; -export const withInteractionIdHeaderSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { +export const withInteractionIdHeaderSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { const { sessionId } = parseUserIdMetadata(ctx.payload.metadata?.user_id); if (sessionId) { ctx.headers.set('x-interaction-id', await sessionUuid(sessionId)); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts index 805a9fb0a8..6fb1800335 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts @@ -30,7 +30,6 @@ const contentHasImage = (content: AnthropicMessagesUserMessage['content'] | Anth export const withVisionHeaderSet = async ( ctx: AnthropicMessagesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { if (ctx.payload.messages.some(message => contentHasImage(message.content))) { diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts index 241d8bf68c..b4cdf58cda 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts @@ -43,7 +43,7 @@ const stripExtensions = (block: T): T => { return withKeysChanged(block, { cache_control: Object.keys(rest).length > 0 ? rest : undefined }); }; -export const withCacheControlExtensionsStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { +export const withCacheControlExtensionsStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { const payload = ctx.payload; const system = Array.isArray(payload.system) diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts index aea6423163..e0ac90c66b 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts @@ -11,7 +11,7 @@ import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; * References: * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L1761 */ -export const withEagerInputStreamingStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { +export const withEagerInputStreamingStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { if (ctx.payload.tools) { ctx.payload.tools = ctx.payload.tools.map(tool => { const { eager_input_streaming: _, ...rest } = tool as typeof tool & { diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts index b03c1e8eef..8088f63f1d 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts +++ b/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts @@ -25,7 +25,6 @@ export interface AnthropicMessagesBoundaryCtx { export type CopilotAnthropicMessagesBoundaryInterceptor = Interceptor< AnthropicMessagesBoundaryCtx, - object, ExecuteResult> >; @@ -34,6 +33,5 @@ export type CopilotAnthropicMessagesBoundaryInterceptor = Interceptor< // only — post-`run()` event-stream inspection is not portable to this result. export type CopilotAnthropicMessagesCountTokensBoundaryInterceptor = Interceptor< AnthropicMessagesBoundaryCtx, - object, Response >; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts index 9b8219d3da..abe74ea8c7 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts +++ b/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts @@ -49,7 +49,7 @@ const isWhitespaceExceeded = ( return false; }; -export const withToolArgumentWhitespaceAborted: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (_invocation, _env, run) => { +export const withToolArgumentWhitespaceAborted: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (_invocation, run) => { const result = await run(); if (result.type !== 'events') return result; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts index 923e1ea4d2..d0a29d4522 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts +++ b/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts @@ -63,7 +63,7 @@ const selectCacheMarkerIndexes = (messages: readonly OpenAIChatCompletionsMessag return [...new Set([...systemIndexes, ...nonSystemIndexes])].sort((a, b) => a - b); }; -export const withCacheControlMarkersAttached: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { +export const withCacheControlMarkersAttached: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, run) => { const marked = new Set(selectCacheMarkerIndexes(ctx.payload.messages)); if (marked.size === 0) return await run(); diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts index 63f9c15f57..4f6cd05217 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts +++ b/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts @@ -46,7 +46,7 @@ const compressInlineImages = async (ctx: OpenAIChatCompletionsBoundaryCtx): Prom // Recompresses every inline base64 image (`data:image/*;base64,...` in an // `image_url` part) in the outgoing OpenAI Chat Completions payload to WebP before // the Copilot upstream call. Remote https image references are left untouched. -export const withInlineImagesCompressed: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { +export const withInlineImagesCompressed: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, run) => { // Finish this nested activation before starting the upstream call. Its // request-local memoizer keys are the full source data URLs, which can be // several megabytes each and must not stay live for the response stream. diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts index 6877390860..917ef12042 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts +++ b/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts @@ -32,7 +32,7 @@ import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts * - https://github.com/microsoft/vscode/blob/fb5e582d1c8edb9ad0a69e50fe6f508a8c095466/extensions/copilot/src/platform/endpoint/node/responsesApi.ts#L419-L468 * - https://github.com/microsoft/vscode/blob/fb5e582d1c8edb9ad0a69e50fe6f508a8c095466/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts#L1453-L1474 */ -export const withInitiatorHeaderSet: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { +export const withInitiatorHeaderSet: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, run) => { const lastMessage = ctx.payload.messages.at(-1); const agentInitiated = lastMessage?.role === 'assistant' || lastMessage?.role === 'tool'; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts index 7c56469aa8..0e932d2ad3 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts +++ b/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts @@ -9,7 +9,7 @@ import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts * References: * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/services/copilot/create-chat-completions.ts#L28-L49 */ -export const withVisionHeaderSet: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { +export const withVisionHeaderSet: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, run) => { const hasImage = ctx.payload.messages.some( message => Array.isArray(message.content) && message.content.some(part => part.type === 'image_url'), ); diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts index 183ae4cef8..0c3d70700c 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts +++ b/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts @@ -13,6 +13,5 @@ export interface OpenAIChatCompletionsBoundaryCtx { export type CopilotOpenAIChatCompletionsBoundaryInterceptor = Interceptor< OpenAIChatCompletionsBoundaryCtx, - object, ExecuteResult> >; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts index 3530fcf9ad..76b051110f 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts @@ -33,7 +33,7 @@ const errorEvent = (): OpenAIResponsesStreamEvent => code: 'api_error', }) as OpenAIResponsesStreamEvent; -export const withToolArgumentWhitespaceAborted: CopilotOpenAIResponsesBoundaryInterceptor = async (_invocation, _env, run) => { +export const withToolArgumentWhitespaceAborted: CopilotOpenAIResponsesBoundaryInterceptor = async (_invocation, run) => { const result = await run(); // Only the streaming generate branch produces events worth inspecting. // The compact branch is a single value envelope; pass it through unchanged. diff --git a/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts b/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts index 55dcb79ee4..1c4d0b8dd3 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts @@ -81,7 +81,6 @@ const compressInlineImages = async (ctx: OpenAIResponsesBoundaryCtx): Promise( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { // Finish this nested activation before starting the upstream call. Its diff --git a/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts b/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts index 52ee48498e..08c6949c1e 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts @@ -13,7 +13,6 @@ import { mapOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses' */ export const withEmptyNamespaceDescriptionsFilled = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { ctx.payload = mapOpenAIResponsesTools(ctx.payload, tool => tool.type === 'namespace' && tool.description === '' diff --git a/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts b/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts index e50c64e5c0..efcb103d6c 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts @@ -14,7 +14,6 @@ import type { OpenAIResponsesBoundaryCtx } from './types.ts'; */ export const withStoreForcedFalse = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { ctx.payload = { ...ctx.payload, store: false }; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts b/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts index 2db859192f..8f3a82e1bf 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts @@ -268,7 +268,7 @@ const normalizeCompactionResult = (response: OpenAIResponsesCompactionResult): O }), }); -export const withCopilotOpenAIResponsesItemIdMembrane: CopilotOpenAIResponsesBoundaryInterceptor = async (ctx, _env, run) => { +export const withCopilotOpenAIResponsesItemIdMembrane: CopilotOpenAIResponsesBoundaryInterceptor = async (ctx, run) => { ctx.payload = restoreInputItemIds(ctx.payload); const result = await run(); if (!result.ok) return result; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts b/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts index b610c17a8f..5806b3e356 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts @@ -24,7 +24,6 @@ import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-resp */ export const withInitiatorHeaderSet = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const lastItem: OpenAIResponsesInputItem | undefined = ctx.payload.input.at(-1); diff --git a/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts b/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts index 0b208d8ad1..d876f505cf 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts @@ -21,7 +21,6 @@ const itemHasImage = (item: OpenAIResponsesInputItem): boolean => { export const withVisionHeaderSet = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { if (ctx.payload.input.some(itemHasImage)) ctx.headers.set('copilot-vision-request', 'true'); diff --git a/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts b/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts index 705782e9d2..3e9acae2cc 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts @@ -51,7 +51,6 @@ export const stripImageGenerationFromPayload = (payload: CanonicalOpenAIResponse export const withImageGenerationStripped = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { stripImageGenerationFromPayload(ctx.payload); diff --git a/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts b/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts index 240e7db7e7..32c83bb73f 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts @@ -14,7 +14,6 @@ import type { OpenAIResponsesBoundaryCtx } from './types.ts'; */ export const withServiceTierStripped = async ( ctx: OpenAIResponsesBoundaryCtx, - _env: object, run: () => Promise, ): Promise => { const { service_tier: _, ...payload } = ctx.payload; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/types.ts b/packages/provider-copilot/src/interceptors/openai-responses/types.ts index 7a985bf188..96ddc9939a 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/types.ts +++ b/packages/provider-copilot/src/interceptors/openai-responses/types.ts @@ -22,6 +22,5 @@ export interface OpenAIResponsesBoundaryCtx { // abort, item-id membrane) inspect the result variant directly. export type CopilotOpenAIResponsesBoundaryInterceptor = Interceptor< OpenAIResponsesBoundaryCtx, - object, ProviderOpenAIResponsesResult >; diff --git a/packages/provider-copilot/src/provider.ts b/packages/provider-copilot/src/provider.ts index bc8e62bf1c..dddf642a05 100644 --- a/packages/provider-copilot/src/provider.ts +++ b/packages/provider-copilot/src/provider.ts @@ -338,8 +338,8 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { headers: new Headers(opts.headers), model, }; - const result = await runInterceptors>>( - ctx, {}, COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY, async () => { + const result = await runInterceptors>>( + ctx, COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY, async () => { const { model: _ignored, ...wireBody } = ctx.payload; return await liftStream(callStreaming(copilotFetchOpenAIChatCompletions, wireBody, signal, rawModel, ctx.headers, parseOpenAIChatCompletionsStream, opts)); }, @@ -365,8 +365,8 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { // compression, vision/initiator headers — applies to both branches // identically. The item-id membrane also normalizes the compact value // envelope, while the whitespace guard only inspects generate streams. - return await runInterceptors( - ctx, {}, COPILOT_OPENAI_RESPONSES_BOUNDARY, async () => { + return await runInterceptors( + ctx, COPILOT_OPENAI_RESPONSES_BOUNDARY, async () => { const { model: _ignored, ...wireBody } = ctx.payload; switch (ctx.action) { case 'generate': { @@ -434,8 +434,8 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { reasoningEffort: anthropicMessagesReasoningEffort(body), fast: body.speed === 'fast', }); - const result = await runInterceptors>>( - ctx, {}, COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, async () => { + const result = await runInterceptors>>( + ctx, COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, async () => { const { model: _ignored, ...wireBody } = ctx.payload; return await liftStream(callStreaming(copilotFetchAnthropicMessages, wireBody, signal, rawModel, headersForAnthropicMessagesCall(ctx.headers, ctx.anthropicBeta), parseAnthropicMessagesStream, opts)); }, @@ -448,8 +448,8 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { context1m: ctx.anthropicBeta.includes(CONTEXT_1M_BETA), reasoningEffort: anthropicMessagesReasoningEffort(body), }); - const response = await runInterceptors( - ctx, {}, COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY, async () => { + const response = await runInterceptors( + ctx, COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY, async () => { const { model: _ignored, ...wireBody } = ctx.payload; const { response } = await call(copilotFetchAnthropicMessagesCountTokens, wireBody, signal, rawModel, headersForAnthropicMessagesCall(ctx.headers, ctx.anthropicBeta), opts); return response; From a7490bf0fe19d98b888daad2ea26f5e18ee7879e Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 20 Aug 2026 00:22:29 +0800 Subject: [PATCH 108/156] chore(migrations): take 0084 back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flag-id rewrite merged into `0083` — one migration for one table — which frees the number this one was moved off. Numbering is contiguous again: `0083` rewrites the stored protocol names, `0084` drops the edge-shaped dump records. --- ...drop_edge_dump_records.sql => 0084_drop_edge_dump_records.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/gateway/migrations/{0085_drop_edge_dump_records.sql => 0084_drop_edge_dump_records.sql} (100%) diff --git a/packages/gateway/migrations/0085_drop_edge_dump_records.sql b/packages/gateway/migrations/0084_drop_edge_dump_records.sql similarity index 100% rename from packages/gateway/migrations/0085_drop_edge_dump_records.sql rename to packages/gateway/migrations/0084_drop_edge_dump_records.sql From 0067da446189eca29296617f3d6c20fb70e3b6ec Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 20 Aug 2026 02:45:03 +0800 Subject: [PATCH 109/156] docs: correct the parameter count in f70d833 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `f70d833` says "the twenty-three `_env: object` parameters". Twenty-three is the count of the inline `(ctx, _env, run)` form; the `_env: object,` form it names is twenty. The commit removed both: 43 parameters in total. Recorded here rather than by rewriting the message, because that commit sits under two merges — rewriting it renumbers 119 commits and drops the review anchors on #476 for one figure. Whoever writes the squash message should use 43. From 0aa0fc00d8004b03955dfa9c8ff1806c7fc4b74e Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 30 Sep 2026 18:37:25 +0800 Subject: [PATCH 110/156] fix: preserve protocol diagnostics across chat stages Record upstream and transformed protocol streams independently, select client frames explicitly, and mask credential headers in the run viewer and exports. Preserve per-call usage through compaction replay and hosted continuations, retain generator terminal values for image sub-requests, and keep cleanup tracking expressed in the run format. --- .../components/requests/detail_test.tsx | 12 ++++ apps/web/src/components/requests/detail.tsx | 8 ++- apps/web/src/i18n/locales/en.ts | 20 ------ apps/web/src/i18n/locales/zh-Hans.ts | 20 ------ .../chat/openai-responses-pipeline_test.ts | 4 +- .../server-tools/image-sub-request_test.ts | 40 ++++++++++++ .../chat/openai-responses/websocket_test.ts | 19 +++--- .../gateway/__tests__/repo/dump-store_test.ts | 18 +++++- .../repo/expiration-sweep-integrity_test.ts | 9 ++- .../scheduled/expiration-sweeps_test.ts | 30 ++++----- .../chat/openai-responses/compact.ts | 5 +- .../server-tools/image-sub-request.ts | 63 ++++++++++++------- .../server-tools/search-sub-request.ts | 37 +++++------ .../src/data-plane/chat/shared/gateway-ctx.ts | 5 +- .../gateway/src/data-plane/pipeline/stages.ts | 2 +- .../src/data-plane/shared/gateway-ctx.ts | 2 +- packages/gateway/src/dump/run-sink.ts | 4 +- .../gateway/src/repo/expiration-sweeps-sql.ts | 54 ++++------------ 18 files changed, 177 insertions(+), 175 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-sub-request_test.ts diff --git a/apps/web/__tests__/components/requests/detail_test.tsx b/apps/web/__tests__/components/requests/detail_test.tsx index 5e779d824c..26a512d582 100644 --- a/apps/web/__tests__/components/requests/detail_test.tsx +++ b/apps/web/__tests__/components/requests/detail_test.tsx @@ -25,6 +25,18 @@ describe('request detail navigation', () => { expect((await screen.findByTestId('body-content')).textContent).toContain('parsed response'); }); + it('masks credential headers in rendered facts', () => { + const secret = 'Bearer private-request-token-1234567890'; + const events = [ + { type: 'object', fromObjectId: 1, nodes: [[['authorization', secret]]] }, + { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'ingress.http.headers': { $: 1 } } }, + ].map(event => `${JSON.stringify(event)}\n`).join(''); + const view = renderInApp(); + expect(view.container.textContent).not.toContain(secret); + expect(view.container.textContent).toContain('authorization'); + expect(view.container.textContent).toContain('••••'); + }); + it('distinguishes an empty run from a search with no matches', () => { renderInApp(); expect(screen.getByText('This run recorded no events.')).toBeTruthy(); diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index b694d80616..f37cd15674 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -4,6 +4,7 @@ import { lazy, Suspense, useMemo, useState } from 'react'; import { RenderedEventList } from './events'; import { downloadRecords } from './export'; import { errorLabel, requestSeverity } from './format'; +import { redactRunHeaders } from './run-redact'; import { renderRunEvents } from './run-render'; import type { CollectedStream } from './stream-render'; import { fluentComponents } from '../../fluent'; @@ -57,9 +58,10 @@ function RecordTiming({ meta }: { meta: DumpMetadata }) { function RunRecordDetail({ record, collected }: { record: DumpRecord; collected: CollectedStream | null }) { const { t } = useTranslation(); - const events = useMemo(() => renderRunEvents(record.events).map(event => ({ + const redacted = useMemo(() => redactRunHeaders(record.events), [record.events]); + const events = useMemo(() => renderRunEvents(redacted).map(event => ({ event: `${event.type} ${event.subject ?? ''}`.trim(), text: event.text, parseError: event.parseError, - })), [record.events]); + })), [redacted]); const failure = errorLabel(record.meta.error) ?? collected?.error; const [view, setView] = useState('run'); return
@@ -74,7 +76,7 @@ function RunRecordDetail({ record, collected }: { record: DumpRecord; collected:
{failure && {failure}}
- {view === 'run' ? {t('dashboard.requests.events', { count: events.length })}} emptyText={t('dashboard.requests.noRunEvents')} /> + {view === 'run' ? {t('dashboard.requests.events', { count: events.length })}} emptyText={t('dashboard.requests.noRunEvents')} /> : }>}
; diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index 6db25a79aa..623feeabe1 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -1027,19 +1027,12 @@ const en = { noKeys: 'No API key has request retention enabled', noKeysDescription: 'Enable request dump retention on an API key to start capturing requests.', apiKeysLink: 'API Keys', - requestBody: 'Request body', run: 'Run', noRunEvents: 'This run recorded no events.', - responseBody: 'Response body', emptyBody: 'Empty body', - truncatedStream: 'No terminal marker was recorded. The assembled content is shown; completeness is unknown.', noStatus: 'No status', - revealValue: 'Reveal value', - hideValue: 'Hide value', - decodeError: 'Could not decode this body ({{error}}); showing raw base64.', streamView: 'Response body view', noMatches: 'No requests match these filters', - formattedBody: 'Formatted body', collected: 'Collected', streamEvent: 'Event', eventParseError: 'Invalid JSON', @@ -1049,19 +1042,6 @@ const en = { expandEvents: 'Expand all events', noEventMatches: 'No matching events', events: 'Events ({{count, number}})', - headers: 'Headers ({{count}})', - raw: 'Raw bytes', - base64: 'Raw bytes (Base64)', - partialCapture: 'Capture ended before EOF. These are the bytes received so far.', - noUpstreamCapture: 'No HTTP upstream capture is available for this record.', - upstreamCall: 'Upstream calls', - callNumber: 'Call {{number}} of {{count}}', - upstreamRequest: 'Upstream request', - upstreamResponse: 'Upstream response', - metadata: 'Details', - diagnostics: 'Issues ({{count}})', - clientRequest: 'Client request', - clientResponse: 'Client response', exportRecord: 'Export record', search: 'Search ID, model, path, upstream, status or error', failuresOnly: 'Failures only', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index 4d5ae92a7b..10ae279976 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -979,19 +979,12 @@ const zhHansCN = { noKeys: '没有 API 密钥启用请求保留', noKeysDescription: '在 API 密钥上启用请求转储保留后,即可开始捕获请求。', apiKeysLink: 'API 密钥', - requestBody: '请求体', run: '运行过程', noRunEvents: '这次运行没有记录到任何事件。', - responseBody: '响应体', emptyBody: '响应体为空', - truncatedStream: '记录中没有结束标记。下面仍展示已解析内容,完整性未知。', noStatus: '无状态码', - revealValue: '显示值', - hideValue: '隐藏值', - decodeError: '无法解码此内容({{error}}),以下显示原始 base64。', streamView: '响应体视图', noMatches: '没有符合筛选条件的请求', - formattedBody: '格式化正文', collected: '聚合结果', streamEvent: '事件', eventParseError: 'JSON 解析失败', @@ -1001,19 +994,6 @@ const zhHansCN = { expandEvents: '展开全部事件', noEventMatches: '没有匹配的事件', events: '事件({{count, number}})', - headers: '请求头 / 响应头({{count}})', - raw: '原始内容', - base64: '原始字节(Base64)', - partialCapture: '采集在 EOF 前结束。这里保留了目前收到的字节。', - noUpstreamCapture: '这条记录没有可用的上游 HTTP 采集。', - upstreamCall: '上游交互', - callNumber: '第 {{number}} / {{count}} 次调用', - upstreamRequest: '上游请求', - upstreamResponse: '上游响应', - metadata: '详情', - diagnostics: '诊断({{count}})', - clientRequest: '客户端请求', - clientResponse: '客户端响应', exportRecord: '导出记录', search: '搜索 ID、模型、路径、上游、状态或错误', failuresOnly: '仅看失败', diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts index cfede8f235..d1d8ac708e 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -497,8 +497,7 @@ describe('the responses chain', () => { expect(answered.id).not.toBe('resp_1'); }); - // A compaction is one envelope rather than a stream: the upstream ran the turn, charged for - // it, and stated the counts in the body. It rides at the same key the frames would have. + // Native compaction counts pass through the same normalization and metering as generation. it('serves an upstream that answered with one envelope as that envelope', async () => { const compaction: OpenAIResponsesCompactionResult = { id: 'resp_compact_1', @@ -520,7 +519,6 @@ describe('the responses chain', () => { expect(facts['response.http.status']).toBe(200); expect(facts['response.chat.openaiResponses.rendered']).toMatchObject({ object: 'response.compaction', usage: compaction.usage }); expect(facts['response.chat.openaiResponses.streamedUsage']).not.toBeNull(); - expect((facts['response.chat.openaiResponses.streamedUsage'] as Deferred | null)).not.toBeNull(); expect((await (facts['response.chat.openaiResponses.streamedUsage'] as Deferred)).billable).toEqual([expect.objectContaining({ quantities: { input_tokens: '900', output_tokens: '40' } })]); }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-sub-request_test.ts new file mode 100644 index 0000000000..0aedbb74bb --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/server-tools/image-sub-request_test.ts @@ -0,0 +1,40 @@ +import { expect, test } from 'vitest'; + +import { runImageGenerationSubRequest } from '../../../../../src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts'; +import type { ServerToolTerminal } from '../../../../../src/data-plane/chat/openai-responses/server-tools/shim.ts'; +import { initDumpBroker, initDumpStore } from '../../../../../src/dump/registry.ts'; +import { openRunDump } from '../../../../../src/dump/run-sink.ts'; +import type { ApiKey } from '../../../../../src/repo/types.ts'; +import { eventsOf, installDumpStubs } from '../../../../dump/test-fixtures.ts'; +import { flushBackground, trackBackground } from '../../../../test-utils/background-tracker.ts'; +import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; + +const apiKey: ApiKey = { id: 'image-key', userId: 1, name: 'Image key', key: 'floway-image-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; + +test('a recorded image lifecycle preserves its terminal result and independent run', async () => { + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); + const terminal: ServerToolTerminal = { item: { type: 'image_generation_call', id: 'image', status: 'completed', result: 'pixels' }, endEvents: [] }; + const progress = { type: 'response.image_generation_call.in_progress', item_id: 'image' }; + const run = await runImageGenerationSubRequest(parent, 'generate', settle => (async function* () { + yield progress; + settle([], false, undefined); + return terminal; + })()); + expect(await run.lifecycle.next()).toEqual({ done: false, value: progress }); + const result = await run.lifecycle.next(); + expect(result.done).toBe(true); + expect(result.value).toBe(terminal); + await run.drain(); + await flushBackground(); + expect(dumps.stored).toHaveLength(1); + const record = dumps.stored[0]!.record; + expect(record.meta.path).toBe('/images/generations'); + const read = createRunReader(); + const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); + expect(decoded.flatMap(event => event?.frames ?? [])).toEqual([{ type: 'event', event: progress }, { type: 'event', event: terminal }]); + expect(new TextDecoder().decode(record.events)).toContain('request.imageGeneration.action'); + expect(new TextDecoder().decode(record.events)).not.toContain('request.imageGeneration.call'); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index 3dae855688..ae639ef524 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -301,18 +301,17 @@ test('OpenAI Responses WebSocket dump responseBytes equals the UTF-8 payload byt ); }); -/** How many frames a run record holds. The format writes each recorded frame as a - * `stream.frame` event, so counting them reads no object table. */ -const recordedFrameCount = (record: Parameters[0]): number => - eventsOf(runRecordOf(record)) - .filter(event => event.type === 'stream.frame') +const recordedFrameCount = (record: Parameters[0]): number => { + const events = eventsOf(runRecordOf(record)); + const selected = events.flatMap(event => { + const facts = event.facts as Record | undefined; + return facts?.['response.chat.clientFrames'] === undefined ? [] : [facts['response.chat.clientFrames'].$stream]; + }).at(-1); + assertExists(selected); + return events.filter(event => event.type === 'stream.frame' && event.streamId === selected) .reduce((total, event) => total + (event.frames as readonly unknown[]).length, 0); +}; -// The record holds one frame per event the turn sent, and the count is what says so. Two -// readers sit on the same iterable — the family's edge, which tees what a client is served -// into the record, and this transport, which reads it to write each event to the socket — so a -// second tee is invisible everywhere except here, and it would make the collected view the -// dashboard replays show the whole turn twice. test('OpenAI Responses WebSocket records one frame per event it sent', async () => { const { apiKey, repo } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts index 425431017b..b1751fb401 100644 --- a/packages/gateway/__tests__/repo/dump-store_test.ts +++ b/packages/gateway/__tests__/repo/dump-store_test.ts @@ -159,7 +159,7 @@ test('FileDumpStore refuses a row that points at no run stream', async () => { .run(); await expect(store.get('key_x', record.meta.id)) - .rejects.toThrow(/dump record 01HZZ000000000000000000NORUN has no run stream to read/u); + .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000NORUN response body descriptor.*type/su); }); test('FileDumpStore.list paginates newest-first with the (createdAt, id) cursor', async () => { @@ -349,3 +349,19 @@ test('FileDumpStore: put + get round-trips through real-filesystem IO', async () await rm(root, { recursive: true, force: true }); } }); + +test('FileDumpStore filters all retained history before applying the page limit', async () => { + const db = await openDb(); + const store = new FileDumpStore(db, new MemoryFileStore()); + const now = Date.now(); + for (let i = 0; i < 5; i++) { + const record = runRecord(`filter-${i}`, now - i); + record.meta.model = i === 4 ? 'needle-model' : 'other'; + record.meta.error = i === 4 ? { kind: 'failed', reason: 'socket reset' } : null; + await store.put('key_x', record); + } + expect((await store.list('key_x', { q: 'needle', limit: 1 })).map(meta => meta.id)).toEqual(['filter-4']); + expect((await store.list('key_x', { q: 'SOCKET', failures: true, limit: 1 })).map(meta => meta.id)).toEqual(['filter-4']); + expect(await store.list('key_x', { q: "' OR 1=1 --", limit: 1 })).toEqual([]); + expect(await store.list('other-key', { q: 'needle', limit: 1 })).toEqual([]); +}); diff --git a/packages/gateway/__tests__/repo/expiration-sweep-integrity_test.ts b/packages/gateway/__tests__/repo/expiration-sweep-integrity_test.ts index 2d467cfbd7..43850b4a54 100644 --- a/packages/gateway/__tests__/repo/expiration-sweep-integrity_test.ts +++ b/packages/gateway/__tests__/repo/expiration-sweep-integrity_test.ts @@ -60,12 +60,17 @@ const insertApiKey = (raw: Awaited>, id: }; const insertDump = (raw: Awaited>, keyId: string, id: string): void => { + raw.run( + `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) + VALUES (?, 'dump-response', ?, 'staged', 0)`, + [`dumps/v1/${keyId}/${id}.run.gz`, JSON.stringify([keyId, id])], + ); raw.run( `INSERT INTO dump_records (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor) - VALUES (?, ?, 1, NULL, '{}', '[]', NULL, NULL, NULL)`, - [keyId, id], + VALUES (?, ?, 1, NULL, '{}', '[]', NULL, NULL, ?)`, + [keyId, id, JSON.stringify({ key: `dumps/v1/${keyId}/${id}.run.gz`, type: 'run' })], ); }; diff --git a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts index 0ba40ee863..b19cf2fc36 100644 --- a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts +++ b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts @@ -282,7 +282,7 @@ test('migration 0066 bounds existing-row discovery and tracks older dump files o } }); -test('expiration backfill rejects malformed legacy dump descriptors with row context', async () => { +test('expiration backfill rejects malformed run dump descriptors with row context', async () => { const db = await createSqlJsDatabase(); try { for (const [filename, sql] of migrationSqlByFilename) { @@ -299,13 +299,11 @@ test('expiration backfill rejects malformed legacy dump descriptors with row con db.run( `INSERT INTO dump_records (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor) - VALUES (?, ?, ?, NULL, '{}', '[]', NULL, ?, NULL)`, - ['key-legacy', recordId, 1_000, JSON.stringify({ key: 'dumps/v1/key-legacy/old.req.gz', type: 'chunks' })], + VALUES (?, ?, ?, NULL, '{}', '[]', NULL, NULL, ?)`, + ['key-legacy', recordId, 1_000, JSON.stringify({ key: 123, type: 'run' })], ); - // Run 0066 and every migration after it so the schema reaches the present - // (incl. columns like `response_upstream_body_descriptor` that the backfill - // SQL selects). The malformed descriptor was seeded before the validate - // trigger existed, so it survives the migration run. + // This malformed run survives edge retirement. Backfill must validate its key before + // it can register a file owner. let started = false; for (const [filename, sql] of migrationSqlByFilename) { if (!started) { @@ -317,7 +315,7 @@ test('expiration backfill rejects malformed legacy dump descriptors with row con const repo = new SqlRepo(wrapSqlJsDatabase(db)); await expect(repo.expirationSweeps.backfillCleanupTracking(500)).rejects.toThrow( - new RegExp(`Invalid dump record key-legacy/${recordId} request body descriptor during expiration backfill.*type`, 'su'), + new RegExp(`Invalid dump record key-legacy/${recordId} response body descriptor during expiration backfill.*key`, 'su'), ); } finally { db.close(); @@ -393,18 +391,20 @@ test('bounded cleanup backfill tracks rows whose API key was hard-deleted', asyn const repo = new SqlRepo(db); await repo.apiKeys.save(key(now)); const recordId = '01K00000000000000000ORPH'; - const fileKey = `dumps/v1/key-a/1970010100/${recordId}.req.gz`; + const fileKey = `dumps/v1/key-a/1970010100/${recordId}.run.gz`; + await db.prepare( + `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) + VALUES (?, 'dump-response', ?, 'staged', 0)`, + ).bind(fileKey, JSON.stringify(['key-a', recordId])).run(); await db.prepare( `INSERT INTO dump_records (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor) - VALUES ('key-a', ?, 1, NULL, '{}', '[]', NULL, ?, NULL)`, - ).bind(recordId, JSON.stringify({ key: fileKey, type: 'bytes' })).run(); + VALUES ('key-a', ?, 1, NULL, '{}', '[]', NULL, NULL, ?)`, + ).bind(recordId, JSON.stringify({ key: fileKey, type: 'run' })).run(); await db.prepare("DELETE FROM api_keys WHERE id = 'key-a'").run(); await db.prepare("DELETE FROM expiration_sweeps WHERE key_id = 'key-a'").run(); await db.prepare('DELETE FROM spilled_files WHERE file_key = ?').bind(fileKey).run(); - // Run 0082 and every migration after it so the backfill SQL finds the - // columns later migrations added (e.g. the upstream body descriptor from - // 0089). The orphan row was seeded before the validate trigger existed. + // The run survives later migrations even though its API key was hard-deleted. let started = false; for (const [filename, sql] of migrationSqlByFilename) { if (!started) { @@ -421,7 +421,7 @@ test('bounded cleanup backfill tracks rows whose API key was hard-deleted', asyn expect(await db.prepare( 'SELECT owner_kind, owner_key, state FROM spilled_files WHERE file_key = ?', ).bind(fileKey).first()).toEqual({ - owner_kind: 'dump-request', + owner_kind: 'dump-response', owner_key: JSON.stringify(['key-a', recordId]), state: 'owned', }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index dbc9a804b3..1b6d65ae03 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -291,10 +291,7 @@ const callOpenAIResponsesCompactUpstream = defineStage< return move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream' as const, frames: syntheticEventsFromCompaction(result.result) }, - // A compaction states its own counts, so what it billed is known before the frames are - // read. It travels as a reading still to come rather than as one already settled, - // because the verdict that goes with it is the edge's to add — and it is declared as - // this run's own, so the run waits for the verdict where it can see it. + // The outer meter reads the synthetic frames after this wire's usage normalizers. [OPENAI_RESPONSES_STREAMED_USAGE]: null, 'response.usage.billable': billable, 'response.http.headers': [], diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts index b0000fd22a..ab281d7c38 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/image-sub-request.ts @@ -1,17 +1,8 @@ -// The image the shim generates, as a run of its own. -// -// Ruling 2-and-6: a server tool's backend call is not part of the turn that asked for it — it is -// an independent run, with its own prologue, its own settlement and its own record. Its stages -// number from 1 like any run's, which is why the record has to be its own: events landing in the -// parent's would collide with the parent's ids and read as one turn entering a stage twice. -// -// What the ending does is unchanged — resolve the pinned image model, dial it with the rate-limit -// retry the hosted tool needs, and produce the lifecycle events the caller splices. What changes -// is where its cost lands: the reading it settles is the one `writeSettlement` writes both rows -// from, so a shim call bills through the same seam every other upstream call does rather than -// through two hand-rolled telemetry calls beside it. +// Hosted backend calls have independent runs, timing and settlement. Their stage and stream +// ids belong to their own object space, so they never enter the parent turn's record. import type { ServerToolLifecycleEvent, ServerToolTerminal } from './shim.ts'; +import type { RunDump } from '../../../../dump/run-sink.ts'; import { consoleLogSink } from '../../../../runtime/log.ts'; import type { BillableEntity, GatewayFacts } from '../../../pipeline/facts.ts'; import type { StreamOutcome } from '../../../pipeline/serve.ts'; @@ -21,12 +12,11 @@ import { settleBillable, writeSettlement } from '../../../pipeline/settlement.ts import type { AttemptState, GatewayCtx } from '../../../shared/gateway-ctx.ts'; import type { PerformanceTelemetryContext } from '../../../shared/telemetry/performance.ts'; import { compose, defer, defineStage, move, run, type Deferred, type Pipeline } from '@floway-dev/pipeline'; +import { eventFrame } from '@floway-dev/protocols/common'; /** What one image call is, and what it comes to. */ export interface ImageSubRequestFacts extends GatewayFacts { - /** The call, already planned: everything the ending needs to dial and nothing about the turn - * that asked for it. */ - 'request.imageGeneration.call': (settle: SettleImageCall) => AsyncGenerator; + 'request.imageGeneration.action': 'generate' | 'edit'; /** The lifecycle the caller splices into its own answer. */ 'response.imageGeneration.lifecycle': AsyncGenerator; /** What the call turned out to cost, once its events have run out — which is after this run @@ -40,15 +30,41 @@ type I = { [P in K]: ImageSubRequestFacts[ * the backend's own response, long after this run handed up. */ export type SettleImageCall = (billable: readonly BillableEntity[], failed: boolean, telemetry: PerformanceTelemetryContext | undefined) => void; +type ImageCall = (settle: SettleImageCall) => AsyncGenerator; +interface ImageServices extends GatewayServices { readonly imageCall: ImageCall } + +const recordLifecycle = (source: AsyncGenerator, dump: RunDump | null): AsyncGenerator => { + if (dump === null) return source; + const recording = dump.openStream(); + const generator = (async function* () { + let completed = false; + try { + for (;;) { + const step = await source.next(); + recording.frame(eventFrame(step.value)); + if (step.done) { + completed = true; + recording.end(); + return step.value; + } + yield step.value; + } + } finally { + if (!completed) await source.return(undefined as never); + } + })(); + return Object.assign(generator, recording.fact); +}; + /** * The ending. It hands up the lifecycle the caller drives, and the reading that lifecycle * settles — the same shape every streaming family hands up, for the same reason: the numbers * arrive with the last event. */ const dialImageGeneration = defineStage< - I<'request.imageGeneration.call'>, + I<'request.imageGeneration.action'>, I<'response.imageGeneration.lifecycle' | 'response.imageGeneration.streamedUsage'> & { 'response.usage.billable': readonly BillableEntity[] }, - GatewayServices + ImageServices >({ name: 'dialImageGeneration', return: { @@ -59,15 +75,14 @@ const dialImageGeneration = defineStage< // Declared as this run's own unfinished work, so the runner waits for it at teardown where // it can see it rather than the reading being started and forgotten. const outcome = defer(new Promise(resolve => { settle = resolve; })); - const call = facts['request.imageGeneration.call']; return move({ ...facts, - 'response.imageGeneration.lifecycle': call((billable, failed, telemetry) => { + 'response.imageGeneration.lifecycle': recordLifecycle(use.imageCall((billable, failed, telemetry) => { // The sample is attributed to this run's own attempt slot, which is what keeps the // turn that asked for the image from having its upstream stamp overwritten. use.gateway.attempt.telemetry = telemetry; settle({ billable, failed }); - }), + }), use.gateway.dump), 'response.imageGeneration.streamedUsage': outcome, // Nothing has been reported when this hands up; what the call turns out to have cost // arrives through the reading above. @@ -77,7 +92,7 @@ const dialImageGeneration = defineStage< }); const imageGenerationSubRequestPipeline: Pipeline< - I<'request.imageGeneration.call'>, + I<'request.imageGeneration.action'>, I<'response.imageGeneration.lifecycle' | 'response.imageGeneration.streamedUsage'> > = compose('imageGenerationSubRequest', [ writeSettlement( @@ -97,7 +112,7 @@ const imageGenerationSubRequestPipeline: Pipeline< export const runImageGenerationSubRequest = async ( parent: GatewayCtx, action: 'generate' | 'edit', - call: ImageSubRequestFacts['request.imageGeneration.call'], + call: ImageCall, ): Promise<{ readonly lifecycle: AsyncGenerator; readonly drain: () => Promise; @@ -115,8 +130,8 @@ export const runImageGenerationSubRequest = async ( const { facts, drain } = await run( imageGenerationSubRequestPipeline, - move({ 'request.imageGeneration.call': call }) as never, - prologue.services as never, + move({ 'request.imageGeneration.action': action }) as never, + { ...prologue.services, imageCall: call } as never, ); return { lifecycle: facts['response.imageGeneration.lifecycle'], diff --git a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts index dfdbcbd100..4c6c2cba61 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/server-tools/search-sub-request.ts @@ -1,16 +1,5 @@ -// The search a shim runs, as a run of its own. -// -// Ruling 2-and-6: a server tool's backend call is not part of the turn that asked for it — it is -// an independent run, with its own prologue, its own settlement and its own record. Its stages -// number from 1 like any run's, which is why the record has to be its own: events landing in the -// parent's would collide with the parent's ids and read as one turn entering a stage twice. -// -// A search settles differently from an image, and the difference is real rather than an -// omission. What the search backend charges is accounted per api key in units no model prices, -// so this run bills no entity — the row it writes names none, which is the settlement stage's -// own statement that a run which measured rather than generated still writes. And no -// `PerformanceOperation` names search, so there is no sample for the performance half to land on. -// What the run buys, then, is the record: a search that ran inside a turn is legible as its own. +// Search backends run independently from the hosted-tool turn. The operations account for +// their own API-key charges; the run records the search result without inventing model usage. import type { GatewayFacts } from '../../../pipeline/facts.ts'; import { prologueFor } from '../../../pipeline/serve.ts'; @@ -22,31 +11,33 @@ import { compose, defineStage, move, run, type Pipeline } from '@floway-dev/pipe /** What one search call is, and what it came to. */ export interface WebSearchSubRequestFacts extends GatewayFacts { - /** The call, already planned: the operations to run and nothing about the turn that asked. */ - 'request.webSearch.call': () => Promise; + 'request.webSearch.action': 'search'; /** What the backend answered, in the shape the hosted-tool item is built from. */ 'response.webSearch.ir': WebSearchCallIR; } type W = { [P in K]: WebSearchSubRequestFacts[P] }; +type SearchCall = () => Promise; +interface SearchServices extends GatewayServices { readonly searchCall: SearchCall } + const runWebSearchCall = defineStage< - W<'request.webSearch.call'>, + W<'request.webSearch.action'>, W<'response.webSearch.ir'> & { 'response.usage.billable': readonly never[] }, - GatewayServices + SearchServices >({ name: 'runWebSearchCall', return: { provides: ['response.webSearch.ir', 'response.usage.billable'] }, - execute: async facts => move({ + execute: async (facts, use) => move({ ...facts, - 'response.webSearch.ir': await facts['request.webSearch.call'](), + 'response.webSearch.ir': await use.searchCall(), // No model was called, so there is no entity to bill: what the backend charges is accounted // per api key by the operations as they run. 'response.usage.billable': [], }) as never, }); -const webSearchSubRequestPipeline: Pipeline, W<'response.webSearch.ir'>> = +const webSearchSubRequestPipeline: Pipeline, W<'response.webSearch.ir'>> = compose('webSearchSubRequest', [ writeSettlement(() => false), runWebSearchCall, @@ -61,7 +52,7 @@ const webSearchSubRequestPipeline: Pipeline, W<'resp */ export const runWebSearchSubRequest = async ( parent: GatewayCtx, - call: WebSearchSubRequestFacts['request.webSearch.call'], + call: SearchCall, ): Promise => { const attempt: AttemptState = { timing: { firstOutputTokenAt: null, upstreamCallStartedAt: null }, telemetry: undefined }; const dump = parent.dump?.openSubRequest({ method: 'POST', path: '/alpha/search' }, false, attempt.timing) ?? null; @@ -75,8 +66,8 @@ export const runWebSearchSubRequest = async ( const { facts, drain } = await run( webSearchSubRequestPipeline, - move({ 'request.webSearch.call': call }) as never, - prologue.services as never, + move({ 'request.webSearch.action': 'search' }) as never, + { ...prologue.services, searchCall: call } as never, ); // Nothing streams out of a search, so the run is over the moment it answers. await drain(); diff --git a/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts index 38313d6183..f2c1a82e77 100644 --- a/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts +++ b/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts @@ -9,9 +9,8 @@ import type { OpenAIResponsesStatefulStore } from '../openai-responses/items/sto // persisting factory, non-OpenAI-Responses sources a no-backing scratchpad store, so // the server-tool shim's request-private state always has a home. Every chat // HTTP/WS entry constructs this via `createChatGatewayCtxFromHono` and threads -// it through serve → narrow → attempt. Passthrough endpoints (OpenAI -// Embeddings / OpenAI Images / OpenAI Audio Transcriptions / OpenAI -// Completions) have no stored-items concept and stay on plain `GatewayCtx`. +// it through candidate selection and protocol stages. Non-chat endpoints +// have no stored-items concept and use `GatewayCtx`. export interface ChatGatewayCtx extends GatewayCtx { readonly affinity: AffinityRequestContext; readonly store: OpenAIResponsesStatefulStore; diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts index 0aace7a78a..85e500cadf 100644 --- a/packages/gateway/src/data-plane/pipeline/stages.ts +++ b/packages/gateway/src/data-plane/pipeline/stages.ts @@ -148,7 +148,7 @@ export const failover = ({ failed, owns, streamedUsage }: Forking) => defineStag // Owned on the way up and handed onward: every attempt's is this stage's to release, // and the one it adopts rides up with ownership going with it. consumes: owns as never, - provides: [...owns, 'response.usage.billable', ...(streamedUsage === undefined ? [] : [streamedUsage])] as never, + provides: [...owns, ...(streamedUsage === undefined ? [] : ['response.usage.billable', streamedUsage])] as never, }, }, execute: async (facts, next, use) => { diff --git a/packages/gateway/src/data-plane/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/shared/gateway-ctx.ts index 7a7db877cb..dfb85371ba 100644 --- a/packages/gateway/src/data-plane/shared/gateway-ctx.ts +++ b/packages/gateway/src/data-plane/shared/gateway-ctx.ts @@ -95,6 +95,6 @@ export const createGatewayCtxFromHono = (c: AuthedContext, opts: CreateGatewayCt // Run the recording's finalize tee on the outgoing Response. Every inbound HTTP // wrapper returns its response through this seam so the dump applies uniformly -// across happy-path, error, and passthrough paths. +// across successful and failed turns. export const finalizeGatewayResponse = (ctx: GatewayCtx, response: Response): Response => ctx.dump?.finalize(response) ?? response; diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index e29c568882..6f1269034a 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -117,7 +117,7 @@ export class RunDump { // --- terminal point --- - // Two input shapes, matching the edge accumulator's seam: + // Two transport-owned completion shapes: // // • `(status, responseBytes)` — the caller already knows what it wrote. // • `(response)` — tees the answer so the client gets bytes flowing while a @@ -213,7 +213,7 @@ export class RunDump { * * `method` and `path` are passed rather than read off a request so a transport * that carries several turns over one connection can name each one as what it - * is — the same reason the edge accumulator takes its method explicitly. + * is. */ export const openRunDump = ( apiKey: ApiKey, diff --git a/packages/gateway/src/repo/expiration-sweeps-sql.ts b/packages/gateway/src/repo/expiration-sweeps-sql.ts index 5cb9028c4d..c18a5035d7 100644 --- a/packages/gateway/src/repo/expiration-sweeps-sql.ts +++ b/packages/gateway/src/repo/expiration-sweeps-sql.ts @@ -19,14 +19,12 @@ interface BackfillRow { interface DumpBackfillRow extends BackfillRow { id: string; - request_body_descriptor: string | null; response_body_descriptor: string | null; - response_upstream_body_descriptor: string | null; } interface DumpFileOwner { fileKey: string; - ownerKind: 'dump-request' | 'dump-response' | 'dump-response-upstream'; + ownerKind: 'dump-response'; keyId: string; recordId: string; } @@ -62,7 +60,7 @@ export class SqlExpirationSweepsRepo implements ExpirationSweepsRepo { private async backfillCleanupSource(source: CleanupBackfillSource, cursor: number, limit: number): Promise { const config = CLEANUP_BACKFILL_SOURCES[source]; const descriptorColumns = source === 'dump_records' - ? ', id, request_body_descriptor, response_body_descriptor, response_upstream_body_descriptor' + ? ', id, response_body_descriptor' : ''; const { results } = await this.db .prepare( @@ -102,35 +100,14 @@ export class SqlExpirationSweepsRepo implements ExpirationSweepsRepo { } private async registerDumpFiles(rows: readonly DumpBackfillRow[]): Promise { - const files: DumpFileOwner[] = rows.flatMap(row => [ - ...(row.request_body_descriptor === null ? [] : [{ - fileKey: decodeDumpBodyDescriptor( - row.request_body_descriptor, - `dump record ${row.key_id}/${row.id} request body descriptor during expiration backfill`, - ).key, - ownerKind: 'dump-request' as const, - keyId: row.key_id, - recordId: row.id, - }]), - ...(row.response_body_descriptor === null ? [] : [{ - fileKey: decodeDumpBodyDescriptor( - row.response_body_descriptor, - `dump record ${row.key_id}/${row.id} response body descriptor during expiration backfill`, - ).key, - ownerKind: 'dump-response' as const, - keyId: row.key_id, - recordId: row.id, - }]), - ...(row.response_upstream_body_descriptor === null ? [] : [{ - fileKey: decodeDumpBodyDescriptor( - row.response_upstream_body_descriptor, - `dump record ${row.key_id}/${row.id} upstream response body descriptor during expiration backfill`, - ).key, - ownerKind: 'dump-response-upstream' as const, - keyId: row.key_id, - recordId: row.id, - }]), - ]); + const files: DumpFileOwner[] = rows.map(row => { + if (row.response_body_descriptor === null) throw new Error(`dump record ${row.key_id}/${row.id} has no run descriptor during expiration backfill`); + return { + fileKey: decodeDumpBodyDescriptor(row.response_body_descriptor, + `dump record ${row.key_id}/${row.id} response body descriptor during expiration backfill`).key, + ownerKind: 'dump-response', keyId: row.key_id, recordId: row.id, + }; + }); if (files.length === 0) return; await this.db .prepare( @@ -145,16 +122,7 @@ export class SqlExpirationSweepsRepo implements ExpirationSweepsRepo { JOIN dump_records AS records ON records.key_id = json_extract(incoming.value, '$.keyId') AND records.id = json_extract(incoming.value, '$.recordId') - WHERE ( - json_extract(incoming.value, '$.ownerKind') = 'dump-request' - AND json_extract(records.request_body_descriptor, '$.key') = json_extract(incoming.value, '$.fileKey') - ) OR ( - json_extract(incoming.value, '$.ownerKind') = 'dump-response' - AND json_extract(records.response_body_descriptor, '$.key') = json_extract(incoming.value, '$.fileKey') - ) OR ( - json_extract(incoming.value, '$.ownerKind') = 'dump-response-upstream' - AND json_extract(records.response_upstream_body_descriptor, '$.key') = json_extract(incoming.value, '$.fileKey') - ) + WHERE json_extract(records.response_body_descriptor, '$.key') = json_extract(incoming.value, '$.fileKey') ON CONFLICT (file_key) DO NOTHING`, ) .bind(JSON.stringify(files)) From 9833a110b1c7e55ef5cac97cbd8dd062b146db03 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 30 Sep 2026 21:30:09 +0800 Subject: [PATCH 111/156] refactor(pipeline): keep stage contracts in their owning modules Reuse the Responses fact contract for native compaction stages and keep simple attempt labels local to their consumers. --- .../normalize-exclusive-cached-tokens.ts | 3 +-- .../chat/openai-responses/compact/call-upstream.ts | 3 ++- .../src/data-plane/chat/openai-responses/compact/dial.ts | 2 +- .../src/data-plane/chat/openai-responses/compact/emit.ts | 3 ++- .../src/data-plane/chat/openai-responses/compact/facts.ts | 4 +--- .../openai-responses/normalize-exclusive-cached-tokens.ts | 3 +-- .../gateway/src/data-plane/chat/shared/attempt-identity.ts | 5 ----- 7 files changed, 8 insertions(+), 15 deletions(-) delete mode 100644 packages/gateway/src/data-plane/chat/shared/attempt-identity.ts diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-exclusive-cached-tokens.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-exclusive-cached-tokens.ts index 26e0821ef6..9cb11e7b0f 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-exclusive-cached-tokens.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-exclusive-cached-tokens.ts @@ -1,7 +1,6 @@ import { asJsonObject } from '../../../shared/json-helpers.ts'; import type { AttemptSelector } from '../../pipeline/facts.ts'; import type { Chat } from '../facts.ts'; -import { attemptIdentity } from '../shared/attempt-identity.ts'; import { withCacheBucketsFolded, type CacheBucketNames } from '../shared/cache-buckets.ts'; import { answerWithFrames, rewritingEvents } from '../shared/frames.ts'; import { defineStage, transform, move } from '@floway-dev/pipeline'; @@ -58,7 +57,7 @@ export const normalizeExclusiveCachedTokensForOpenAIChatCompletions = defineStag }, response: facts => { const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); - const identity = attemptIdentity(attempt); + const identity = `${attempt.upstreamId}/${attempt.modelId}`; const answer = answerWithFrames( facts['response.chat.openaiChatCompletions'], frames => rewritingEvents(frames, chunk => foldOpenAIChatCompletionsUsage(chunk, declaredExclusive, identity)), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts index c926445cbc..a7e5619c74 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts @@ -1,4 +1,5 @@ -import type { Fields, Compacted } from './facts.ts'; +import type { Fields } from '../facts.ts'; +import type { Compacted } from './facts.ts'; import { bodyForAttempt } from '../../../pipeline/attempt-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../../../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../../../shared/upstream-call-options.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts index 1d3941fbee..27b18e5e1d 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts @@ -1,7 +1,7 @@ -import type { Fields } from './facts.ts'; import type { ChatServices } from '../../services.ts'; import type { ChatWire } from '../../wire.ts'; import { simulatesCompaction } from '../compaction-policy.ts'; +import type { Fields } from '../facts.ts'; import { OPENAI_RESPONSES_STREAMED_USAGE } from '../facts.ts'; import { defineStage } from '@floway-dev/pipeline'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts index 4748cc104f..13074d0e7f 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts @@ -1,4 +1,5 @@ -import type { Compacted, Fields } from './facts.ts'; +import type { Fields } from '../facts.ts'; +import type { Compacted } from './facts.ts'; import { recordStream } from '../../../../dump/run-sink.ts'; import { isFailure, renderFailure, mintedErrorEnvelope } from '../../../pipeline/facts.ts'; import type { StreamOutcome } from '../../../pipeline/serve.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts index ed085e44c9..0af1e4ee01 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts @@ -1,7 +1,5 @@ import type { Failure } from '../../../pipeline/facts.ts'; -import type { OpenAIResponsesFacts } from '../facts.ts'; - -export type Fields = { [P in K]: OpenAIResponsesFacts[P] }; +import type { Fields } from '../facts.ts'; /** What a compaction wire hands up. A compaction is a stream by the time it leaves either * ending — the envelope is expanded into the events the stateful half reads — so the value diff --git a/packages/gateway/src/data-plane/chat/openai-responses/normalize-exclusive-cached-tokens.ts b/packages/gateway/src/data-plane/chat/openai-responses/normalize-exclusive-cached-tokens.ts index ca8e59592e..906779f451 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/normalize-exclusive-cached-tokens.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/normalize-exclusive-cached-tokens.ts @@ -1,7 +1,6 @@ import { asJsonObject, type JsonObject } from '../../../shared/json-helpers.ts'; import type { AttemptSelector } from '../../pipeline/facts.ts'; import type { Chat } from '../facts.ts'; -import { attemptIdentity } from '../shared/attempt-identity.ts'; import { withCacheBucketsFolded, type CacheBucketNames } from '../shared/cache-buckets.ts'; import { answerWithFrames, rewritingEvents } from '../shared/frames.ts'; import { defineStage, transform, move } from '@floway-dev/pipeline'; @@ -46,7 +45,7 @@ export const normalizeExclusiveCachedTokensForOpenAIResponses = defineStage< }, response: facts => { const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); - const identity = attemptIdentity(attempt); + const identity = `${attempt.upstreamId}/${attempt.modelId}`; const answer = answerWithFrames( facts['response.chat.openaiResponses'], frames => rewritingEvents(frames, event => foldOpenAIResponsesUsage(event, declaredExclusive, identity)), diff --git a/packages/gateway/src/data-plane/chat/shared/attempt-identity.ts b/packages/gateway/src/data-plane/chat/shared/attempt-identity.ts deleted file mode 100644 index a5e4b97d74..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/attempt-identity.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { AttemptSelector } from '../../pipeline/facts.ts'; - -/** How the attempt whose usage is being read is named in the errors the fold raises: an - * operator is told which upstream and which model to set the flag on. */ -export const attemptIdentity = (attempt: AttemptSelector): string => `${attempt.upstreamId}/${attempt.modelId}`; From 65cf9f5cb2a751b5211d848b259259468509b864 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 30 Sep 2026 21:39:50 +0800 Subject: [PATCH 112/156] refactor(pipeline): use hosted-tool vocabulary throughout the stages Align dispatcher contracts, private state, diagnostics and tests with the hosted-tool execution model. Clarify candidate isolation and the request/response traversal order. --- .../image-generation-integration_test.ts | 50 +- .../hosted-tools/image-generation_test.ts | 16 +- .../hosted-tools/web-search_test.ts | 42 +- .../openai-responses/hosted-tools_test.ts | 1088 ++++++++--------- .../chat/openai-chat-completions/wire.ts | 2 +- .../openai-responses/begin-stored-attempt.ts | 10 +- .../chat/openai-responses/hosted-tools.ts | 12 +- .../hosted-tools/image-generation.ts | 36 +- .../hosted-tools/image-sub-request.ts | 2 +- .../hosted-tools/search-sub-request.ts | 2 +- .../openai-responses/hosted-tools/shared.ts | 20 +- .../openai-responses/hosted-tools/types.ts | 6 +- .../hosted-tools/web-search.ts | 54 +- 13 files changed, 668 insertions(+), 672 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts index 25222e4fa1..f5e5958d50 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts @@ -13,10 +13,10 @@ import { type EventResult, type ExecuteResult, type FlagId, type OpenAIImagesEdi import { assert, assertEquals, assertStringIncludes, stubModelCandidate } from '@floway-dev/test-utils'; // Dirty integration harness: mock the model registry so the image backend is a -// pair of in-test stubs, then drive the whole shim (function-tool rewrite, +// pair of in-test stubs, then drive the whole dispatcher (function-tool rewrite, // ReAct loop, dispatch, streaming relay, output feedback) over a scripted // upstream. The stubs record every backend call so a test can assert what the -// shim actually forwarded — most importantly that an image generated in turn 1 +// dispatcher actually forwarded — most importantly that an image generated in turn 1 // is re-collected as an edit source in turn 2. interface BackendStub { @@ -35,7 +35,7 @@ interface BackendStub { const stub = vi.hoisted((): BackendStub => ({ generationsCalls: [], editsRequests: [], nextGenerations: [], nextEdits: [], nextResolutionOverride: null })); // Assigned per test in beforeEach and captured so the perf-attribution test -// can read `repo.performance.listAll()` after the shim completes. +// can read `repo.performance.listAll()` after the dispatcher completes. let repo: InMemoryRepo; const defaultCandidates = vi.hoisted(() => () => [{ @@ -90,7 +90,7 @@ vi.mock('../../../../../src/data-plane/providers/resolution.ts', () => ({ // Imported AFTER vi.mock so the mocked registry is in effect. const { imageGenerationHostedTool } = await import('../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts'); -const shim = driveHostedToolStage([imageGenerationHostedTool]); +const dispatcher = driveHostedToolStage([imageGenerationHostedTool]); const MODEL_IDENTITY = { model: 'orchestrator', upstream: 'u', modelKey: 'orchestrator', pricing: null }; @@ -232,7 +232,7 @@ beforeEach(async () => { test('generates an image end-to-end and emits the native lifecycle', async () => { stub.nextGenerations = [jsonResponse('R0VO')]; // "GEN" - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { size: '1024x1024', quality: 'low', }), gatewayCtx(), scriptedRun([ @@ -260,7 +260,7 @@ test('generates an image from an additional_tools declaration without moving the ]); delete invocation.payload.tools; - const result = await shim(invocation, gatewayCtx(), scriptedRun([ + const result = await dispatcher(invocation, gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('done'), ])); @@ -280,7 +280,7 @@ test('invalid image generation in tool_search_output reports its input path', as }]); delete invocation.payload.tools; - const result = await shim(invocation, gatewayCtx(), async () => { + const result = await dispatcher(invocation, gatewayCtx(), async () => { throw new Error('Invalid request reached upstream'); }); assert(result.type === 'api-error'); @@ -294,7 +294,7 @@ test('restores a forced hosted choice when the terminal upstream echo omits it', const invocation = makeCtx([], 'generate', { quality: 'high' }, hostedChoice); const hostedTool = invocation.payload.tools![0]; const replacement = { type: 'function', name: 'image_generation', parameters: {}, strict: false } as OpenAIResponsesTool; - const result = await shim(invocation, gatewayCtx(), scriptedRun([ + const result = await dispatcher(invocation, gatewayCtx(), scriptedRun([ withResponseEcho(callTurn(0, 'call_1', 'a cat'), [replacement], { type: 'function', name: 'image_generation' }), withResponseEcho(messageTurn('done'), [replacement]), ])); @@ -313,7 +313,7 @@ test('keeps the top-level hosted echo when a later input declaration changes the ], 'generate', { quality: 'low' }); const topLevelTool = invocation.payload.tools![0]; const replacement = { type: 'function', name: 'image_generation', parameters: {}, strict: false } as OpenAIResponsesTool; - const result = await shim(invocation, gatewayCtx(), scriptedRun([ + const result = await dispatcher(invocation, gatewayCtx(), scriptedRun([ withResponseEcho(callTurn(0, 'call_1', 'a cat'), [replacement]), withResponseEcho(messageTurn('done'), [replacement]), ])); @@ -331,7 +331,7 @@ test('relays real partial_image frames when partial_images > 0', async () => { JSON.stringify({ type: 'image_generation.partial_image', partial_image_index: 1, b64_json: 'UDE=' }), JSON.stringify({ type: 'image_generation.completed', b64_json: 'RklO' }), ])]; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { partial_images: 2 }), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { partial_images: 2 }), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('done'), ])); @@ -347,13 +347,13 @@ test('relays real partial_image frames when partial_images > 0', async () => { }); test('an image generated in turn 1 is re-collected as an edit source in turn 2', async () => { - // Turn 1 generates "AAAA"; the shim feeds it back as an input_image. Turn 2's + // Turn 1 generates "AAAA"; the dispatcher feeds it back as an input_image. Turn 2's // call must therefore resolve to an EDIT whose image[] part carries those // exact bytes — proving the dispatcher re-collects the live input rather than // a frozen registration-time snapshot. stub.nextGenerations = [jsonResponse('QUFBQQ==')]; // "AAAA" stub.nextEdits = [jsonResponse('QkJCQg==')]; // "BBBB" - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw then border it' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw then border it' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), callTurn(0, 'call_2', 'add a black border'), messageTurn('done'), @@ -398,7 +398,7 @@ test('a prefetched remote edit source remains visible to orchestration and is re return await baseRun(); }; - await drain(await shim(invocation, gatewayCtx(), run)); + await drain(await dispatcher(invocation, gatewayCtx(), run)); assertEquals(fetched, ['https://example.com/source.png']); assertEquals(orchestratorImageUrl, 'https://example.com/source.png'); @@ -419,7 +419,7 @@ test('mask-only GIF edit transcodes one shared image and mask to WebP', async () }); const gif = 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; stub.nextEdits = [jsonResponse('RURJVA==')]; - const result = await shim(makeCtx([], 'edit', { + const result = await dispatcher(makeCtx([], 'edit', { input_image_mask: { image_url: `data:image/gif;base64,${gif}` }, }), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'edit from the mask'), @@ -450,7 +450,7 @@ test('identical GIF source and mask share one transcode', async () => { }); const gif = 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; stub.nextEdits = [jsonResponse('RURJVA==')]; - const result = await shim(makeCtx([{ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: [{ type: 'input_image', image_url: `data:image/gif;base64,${gif}`, detail: 'auto' }], }], 'edit', { @@ -475,7 +475,7 @@ test('image transcoding failure becomes a terminal image tool failure', async () compressToWebp: () => Promise.reject(new Error('codec down')), }); const gif = 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; - const result = await shim(makeCtx([{ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: [{ type: 'input_image', image_url: `data:image/gif;base64,${gif}`, detail: 'auto' }], }], 'edit'), gatewayCtx(), scriptedRun([ @@ -501,7 +501,7 @@ test('retries on 429 and surfaces the eventual success', async () => { rateLimitResponse(1), jsonResponse('T0s='), ]; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('done'), ])); @@ -522,7 +522,7 @@ test('gives up after MAX_RATE_LIMIT_RETRIES on persistent 429 and surfaces a fai rateLimitResponse(1), rateLimitResponse(1), ]; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -544,7 +544,7 @@ test('does not retry non-rate-limit upstream failures', async () => { { status: 400, headers: new Headers({ 'content-type': 'application/json' }) }, ), ]; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -565,7 +565,7 @@ test('does not retry non-rate-limit upstream failures', async () => { test('resolveImageCandidate renders model_not_found when no upstream knows the model id', async () => { stub.nextResolutionOverride = { candidates: [], sawModel: false, failedUpstreams: [] }; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -585,7 +585,7 @@ test('resolveImageCandidate renders model_not_supported when sawModel=true but n // Mirrors the resolver's "id exists in some catalog but the kind filter // dropped it" signal — sawModel=true, candidates=[]. stub.nextResolutionOverride = { candidates: [], sawModel: true, failedUpstreams: [] }; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -614,7 +614,7 @@ test('resolveImageCandidate renders model_not_supported when image-kind candidat sawModel: true, failedUpstreams: [], }; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -631,7 +631,7 @@ test('resolveImageCandidate renders model_not_supported when image-kind candidat test('an image sub-call records its own perf row attributed to the image backend, leaving the outer attempt untouched', async () => { stub.nextGenerations = [jsonResponse('R0VO', { input_tokens: 9, output_tokens: 4, total_tokens: 13 })]; - // Real scheduler: capture the promises the shim fires so the test can + // Real scheduler: capture the promises the dispatcher fires so the test can // await them before querying the repo (the default no-op scheduler in // `gatewayCtx()` would drop the recordSample write). const pending: Promise[] = []; @@ -639,7 +639,7 @@ test('an image sub-call records its own perf row attributed to the image backend wantsStream: true, backgroundScheduler: p => { pending.push(p); }, }); - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }]), ctx, scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }]), ctx, scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('here it is'), ])); @@ -661,7 +661,7 @@ test('an image sub-call records its own perf row attributed to the image backend const imageUsage = usageRows.filter(row => row.model === 'gpt-image-2'); assertEquals(imageUsage.length, 1); assertEquals(imageUsage[0]!.upstream, 'u'); - // The image shim runs on a local AttemptState distinct from the outer + // The image dispatcher runs on a local AttemptState distinct from the outer // OpenAI Responses turn's — no image-call stamps may leak onto ctx.attempt. assertEquals(ctx.attempt.timing.upstreamCallStartedAt, null); assertEquals(ctx.attempt.timing.firstOutputTokenAt, null); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts index f8861f4d95..ed3aa186b1 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts @@ -14,7 +14,7 @@ import { parseRetryAfterMs, prepareImageGenerationConfig, resolveImageOperation, - SHIM_TOOL_NAME, + FUNCTION_TOOL_NAME, synthesizeImageGenerationCallId, transformInputItemsForImageGeneration, } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts'; @@ -239,9 +239,9 @@ test('prepareImageGenerationConfig decodes image_url masks and reports file_id a // ── buildImageGenerationFunctionTool ── test('buildImageGenerationFunctionTool exposes only an optional prompt and is non-strict', () => { - const tool = buildImageGenerationFunctionTool({ type: 'image_generation' }, SHIM_TOOL_NAME); + const tool = buildImageGenerationFunctionTool({ type: 'image_generation' }, FUNCTION_TOOL_NAME); assertEquals(tool.type, 'function'); - assertEquals(tool.name, SHIM_TOOL_NAME); + assertEquals(tool.name, FUNCTION_TOOL_NAME); assertEquals(tool.strict, false); const params = tool.parameters as { properties: Record; required: unknown[]; additionalProperties: unknown }; assertEquals(Object.keys(params.properties), ['prompt']); @@ -383,7 +383,7 @@ test('a request inspector reuses decoded bytes after generated results become re output_format: 'jpeg', }; const initial = inspect([generated]); - const replay = inspect(transformInputItemsForImageGeneration([generated], SHIM_TOOL_NAME)); + const replay = inspect(transformInputItemsForImageGeneration([generated], FUNCTION_TOOL_NAME)); assert(initial.sources[0] === replay.sources[0]); }); @@ -396,7 +396,7 @@ test('auto generation pivots to edit when a generated image is fed back', () => const replayInput = transformInputItemsForImageGeneration([ { type: 'image_generation_call', id: 'ig_replay', status: 'completed', result: PNG_B64 }, - ], SHIM_TOOL_NAME); + ], FUNCTION_TOOL_NAME); const replay = resolveImageOperation(config, inspect(replayInput)); assert(replay.ok); assertEquals(replay.action, 'edit'); @@ -654,7 +654,7 @@ test('imageGenerationHostedTool fetches repeated remote edit sources once', asyn assert(result.type === 'active'); assertEquals(urls, ['https://example.com/source.png']); - const transformed = result.transformItems?.(input, SHIM_TOOL_NAME); + const transformed = result.transformItems?.(input, FUNCTION_TOOL_NAME); assert(transformed?.[0].type === 'message' && Array.isArray(transformed[0].content)); const first = transformed[0].content[0]; assert(first.type === 'input_image'); @@ -952,7 +952,7 @@ test('image dispatcher exposes a newly introduced invalid edit source as an inva }; assertThrows( () => result.hosted?.dispatcher({ - intercepted: { callId: 'call_live_invalid', name: SHIM_TOOL_NAME, arguments: { prompt: 'edit it' } }, + intercepted: { callId: 'call_live_invalid', name: FUNCTION_TOOL_NAME, arguments: { prompt: 'edit it' } }, loopState: { iterationCount: 2, remainingToolCalls: undefined }, }), Error, @@ -966,7 +966,7 @@ test('image dispatch budget caps real backend calls per response, not ReAct turn const result = await imageGenerationHostedTool(makeCtx({ tools: [{ type: 'image_generation', action: 'generate' }] }), gatewayCtx()); assert(result.type === 'active' && result.hosted !== undefined); const dispatch = result.hosted.dispatcher; - const intercepted = { callId: 'c', name: SHIM_TOOL_NAME, argumentsJson: '{}', arguments: { prompt: 'x' } }; + const intercepted = { callId: 'c', name: FUNCTION_TOOL_NAME, argumentsJson: '{}', arguments: { prompt: 'x' } }; const loopState = { iterationCount: 1, remainingToolCalls: undefined }; for (let i = 0; i < 10; i++) { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/web-search_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/web-search_test.ts index 0775c451ab..70720561a2 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/web-search_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/web-search_test.ts @@ -3,10 +3,10 @@ import { test } from 'vitest'; import { resolveHostedToolName } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/shared.ts'; import { alphaSearchSettingsFromHosted, - buildShimFunctionTool, + buildFunctionTool, isHostedWebSearchTool, - prepareToolsForShim, - SHIM_TOOL_NAME, + prepareTools, + FUNCTION_TOOL_NAME, synthesizeWebSearchCallId, transformInputItemsForWebSearch, WEB_SEARCH_HOSTED_TYPES, @@ -16,7 +16,7 @@ import { assertLocalWebSearchSupport, findMatches, formatMatches, isUrlAllowed, import type { OpenAIResponsesHostedTool, OpenAIResponsesTool, OpenAIResponsesWebSearchAction, OpenAIResponsesWebSearchResult } from '@floway-dev/protocols/openai-responses'; import { assert, assertEquals, assertThrows } from '@floway-dev/test-utils'; -// ── Shim call argument parsing (parseWebSearchOperations) ── +// ── Dispatcher call argument parsing (parseWebSearchOperations) ── const opsOf = (args: Record | null): WebSearchOperation[] => { const parsed = parseWebSearchOperations(args); @@ -25,7 +25,7 @@ const opsOf = (args: Record | null): WebSearchOperation[] => { }; test('injected web_search function exposes the complete OpenAI alpha-search command shape', () => { - const functionTool = buildShimFunctionTool({ type: 'web_search' }, SHIM_TOOL_NAME); + const functionTool = buildFunctionTool({ type: 'web_search' }, FUNCTION_TOOL_NAME); const properties = (functionTool.parameters as { properties: Record }).properties; assertEquals(Object.keys(properties), [ 'search_query', @@ -382,11 +382,11 @@ test('formatMatches: multi-match output uses Match N: headers', () => { // ── Tool detection, filter prep, and name resolution ── -const SHIM_TOOL = SHIM_TOOL_NAME; +const FUNCTION_NAME = FUNCTION_TOOL_NAME; const hostedVariants = ['web_search', 'web_search_2025_08_26', 'web_search_preview', 'web_search_preview_2025_03_11'] as const; const prepare = (tools: OpenAIResponsesTool[]) => { - const result = prepareToolsForShim(tools); + const result = prepareTools(tools); assert(result.ok); return { filters: result.filters }; }; @@ -399,12 +399,12 @@ test('isHostedWebSearchTool recognizes every hosted variant', () => { }); for (const type of hostedVariants) { - test(`prepareToolsForShim accepts ${type} and extracts default filters`, () => { + test(`prepareTools accepts ${type} and extracts default filters`, () => { assertEquals(prepare([{ type } as OpenAIResponsesTool]).filters, { maxResults: 20 }); }); } -test('prepareToolsForShim extracts filters, user_location, and context size', () => { +test('prepareTools extracts filters, user_location, and context size', () => { const { filters } = prepare([{ type: 'web_search', filters: { allowed_domains: ['a.com'], blocked_domains: ['b.com'] }, @@ -417,7 +417,7 @@ test('prepareToolsForShim extracts filters, user_location, and context size', () assertEquals(filters.maxResults, 40); }); -test('prepareToolsForShim selects the last web_search declaration as one configuration', () => { +test('prepareTools selects the last web_search declaration as one configuration', () => { const { filters } = prepare([ { type: 'web_search', @@ -439,22 +439,22 @@ test('prepareToolsForShim selects the last web_search declaration as one configu }); }); -test('prepareToolsForShim passes through with empty filters when no hosted web_search exists', () => { +test('prepareTools passes through with empty filters when no hosted web_search exists', () => { const fn: OpenAIResponsesTool = { type: 'function', name: 'foo', parameters: {}, strict: false }; assertEquals(prepare([fn]).filters, {}); }); test('resolveHostedToolName returns the first free sequential name', () => { - assertEquals(resolveHostedToolName(SHIM_TOOL, []), SHIM_TOOL); - assertEquals(resolveHostedToolName(SHIM_TOOL, [{ type: 'function', name: SHIM_TOOL, parameters: {}, strict: false }]), `${SHIM_TOOL}_2`); - assertEquals(resolveHostedToolName(SHIM_TOOL, [ - { type: 'function', name: SHIM_TOOL, parameters: {}, strict: false }, - { type: 'custom', name: `${SHIM_TOOL}_2` }, - ]), `${SHIM_TOOL}_3`); + assertEquals(resolveHostedToolName(FUNCTION_NAME, []), FUNCTION_NAME); + assertEquals(resolveHostedToolName(FUNCTION_NAME, [{ type: 'function', name: FUNCTION_NAME, parameters: {}, strict: false }]), `${FUNCTION_NAME}_2`); + assertEquals(resolveHostedToolName(FUNCTION_NAME, [ + { type: 'function', name: FUNCTION_NAME, parameters: {}, strict: false }, + { type: 'custom', name: `${FUNCTION_NAME}_2` }, + ]), `${FUNCTION_NAME}_3`); }); -test('prepareToolsForShim rejects invalid hosted fields', () => { - const result = prepareToolsForShim([{ type: 'web_search', search_context_size: 'huge' } as unknown as OpenAIResponsesTool]); +test('prepareTools rejects invalid hosted fields', () => { + const result = prepareTools([{ type: 'web_search', search_context_size: 'huge' } as unknown as OpenAIResponsesTool]); assertEquals(result.ok, false); }); @@ -478,7 +478,7 @@ const makePrivatePayload = ( }); test('transformInputItemsForWebSearch replays the upstream function_call verbatim when a private payload exists', () => { - // One wsc maps 1:1 to one shim call. The shim's + // One wsc maps 1:1 to one dispatcher call. The dispatcher's // jsonrepair-canonical args and the per-op output are persisted on // this single row. const payload = makePrivatePayload( @@ -506,7 +506,7 @@ test('transformInputItemsForWebSearch replays the upstream function_call verbati }); test('transformInputItemsForWebSearch replays each echoed wsc independently (one pair per wsc)', () => { - // Two distinct shim calls (different upstream call_ids) → two replay + // Two distinct dispatcher calls (different upstream call_ids) → two replay // pairs, one per wsc. const p1 = makePrivatePayload('call_u1', '{"search_query":[{"q":"q1"}]}', { type: 'search', queries: ['q1'] }, [{ type: 'text_result', url: 'u1', title: 't1', snippet: 'body1' }]); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts index 1b8e8f36b5..8eb9b9f54d 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts @@ -4,7 +4,7 @@ import { driveHostedToolStage } from './hosted-tools/drive.ts'; import { imageGenerationHostedTool } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts'; import { consumeTurnStreaming, createMergeState, materializeAccumulatedOutput, parseHostedToolArguments, resolveHostedToolName, sumUsage } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/shared.ts'; import { type InterceptedFunctionCall, type HostedToolResultSlot, type TurnSummary, type UpstreamTerminal } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/types.ts'; -import { SHIM_TOOL_NAME, webSearchHostedTool } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/web-search.ts'; +import { FUNCTION_TOOL_NAME, webSearchHostedTool } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/web-search.ts'; import { createNonOpenAIResponsesSourceStore } from '../../../../src/data-plane/chat/openai-responses/items/store.ts'; import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; import { resolveAlphaSearchDispatcher } from '../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; @@ -216,7 +216,7 @@ const mkResponseIncomplete = ( }, }); -// Integration tests for the OpenAI Responses web_search shim. Internal helpers +// Integration tests for the OpenAI Responses web_search dispatcher. Internal helpers // are unit-tested through focused web-search hosted-tool suites; this file covers // activation gating, tool rewrite, multi-turn loop, downstream stream // merge invariants, and error propagation. @@ -355,8 +355,8 @@ const collectFrames = async (iter: AsyncIterable): Promise => { // The stage streams lazily: backend calls, the payload each descent sends, and the descents // themselves all happen inside the events generator. Tests asserting on side-effects MUST drain // the events stream first. -const runShimAndDrain = async ( - shim: ReturnType, +const runHostedToolsAndDrain = async ( + dispatcher: ReturnType, inv: OpenAIResponsesInvocation, gatewayCtx: ChatGatewayCtx, run: () => Promise>>, @@ -364,7 +364,7 @@ const runShimAndDrain = async ( result: ExecuteResult>; frames: ProtocolFrame[]; }> => { - const result = await shim(inv, gatewayCtx, run); + const result = await dispatcher(inv, gatewayCtx, run); const frames: ProtocolFrame[] = result.type === 'events' ? await collectFrames(result.events) : []; @@ -401,7 +401,7 @@ const scriptedRun = (turns: ScriptedTurn[]): ScriptedRun => { }; // `outputIndex` is the UPSTREAM index (which resets to 0 every run()); -// the shim assigns its own downstream index. +// the dispatcher assigns its own downstream index. const messageTurn = (text: string, outputIndex = 0): ScriptedTurn => [ mkResponseCreated(), mkResponseInProgress(), @@ -425,13 +425,13 @@ const fcTurn = ( ]; const searchCallTurn = (outputIndex: number, callId: string, query: string): ScriptedTurn => - fcTurn(outputIndex, callId, SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: query }] })); + fcTurn(outputIndex, callId, FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: query }] })); const openCallTurn = (outputIndex: number, callId: string, url: string): ScriptedTurn => - fcTurn(outputIndex, callId, SHIM_TOOL_NAME, JSON.stringify({ open: [{ ref_id: url }] })); + fcTurn(outputIndex, callId, FUNCTION_TOOL_NAME, JSON.stringify({ open: [{ ref_id: url }] })); const findCallTurn = (outputIndex: number, callId: string, url: string, pattern: string): ScriptedTurn => - fcTurn(outputIndex, callId, SHIM_TOOL_NAME, JSON.stringify({ find: [{ ref_id: url, pattern }] })); + fcTurn(outputIndex, callId, FUNCTION_TOOL_NAME, JSON.stringify({ find: [{ ref_id: url, pattern }] })); const eventPayloads = (frames: ProtocolFrame[]): OpenAIResponsesStreamEvent[] => frames.filter(f => f.type === 'event').map(f => (f as { type: 'event'; event: OpenAIResponsesStreamEvent }).event); @@ -442,9 +442,9 @@ const outputItemDoneEvents = (frames: ProtocolFrame[ // ── Activation gating ────────────────────────────────────────────────── -test('shim no-ops when targetApi=openaiResponses and flag is off', async () => { +test('dispatcher no-ops when targetApi=openaiResponses and flag is off', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(), @@ -452,56 +452,56 @@ test('shim no-ops when targetApi=openaiResponses and flag is off', async () => { const originalPayload = inv.payload; const script = scriptedRun([messageTurn('hi back')]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(script.callCount(), 1); - // Payload reference unchanged: the shim did not rewrite tools. + // Payload reference unchanged: the dispatcher did not rewrite tools. assertEquals(inv.payload, originalPayload); assertEquals(backend.calls.length, 0); assertEquals(result.type, 'events'); }); -test('shim activates when targetApi=openaiResponses and flag is on', async () => { +test('dispatcher activates when targetApi=openaiResponses and flag is on', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']), }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.length, 1); assertEquals(inv.payload.tools?.[0].type, 'function'); - assertEquals((inv.payload.tools?.[0] as { name: string }).name, SHIM_TOOL_NAME); + assertEquals((inv.payload.tools?.[0] as { name: string }).name, FUNCTION_TOOL_NAME); }); -test('shim activates when targetApi=anthropicMessages and flag is off', async () => { +test('dispatcher activates when targetApi=anthropicMessages and flag is off', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'anthropicMessages', enabledFlags: new Set() }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.length, 1); }); -test('shim activates when targetApi=openai-chat-completions and flag is off', async () => { +test('dispatcher activates when targetApi=openai-chat-completions and flag is off', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiChatCompletions', enabledFlags: new Set() }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.length, 1); }); -test('shim no-ops when no hosted web_search tool is present', async () => { +test('dispatcher no-ops when no hosted web_search tool is present', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'function', name: 'other', parameters: { type: 'object' }, strict: false }], @@ -509,7 +509,7 @@ test('shim no-ops when no hosted web_search tool is present', async () => { }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(script.callCount(), 1); assertEquals(backend.calls.length, 0); @@ -521,27 +521,27 @@ test('shim no-ops when no hosted web_search tool is present', async () => { const hostedAliasTypes = ['web_search', 'web_search_2025_08_26', 'web_search_preview', 'web_search_preview_2025_03_11'] as const; for (const type of hostedAliasTypes) { - test(`shim rewrites hosted ${type} alias into the shim function tool`, async () => { + test(`dispatcher rewrites hosted ${type} alias into the dispatcher function tool`, async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type }] }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.length, 1); - assertEquals((inv.payload.tools?.[0] as { name?: string }).name, SHIM_TOOL_NAME); + assertEquals((inv.payload.tools?.[0] as { name?: string }).name, FUNCTION_TOOL_NAME); }); } // ── tool_choice rewrite × 4 hosted-type values ───────────────────────── for (const type of hostedAliasTypes) { - test(`shim rewrites tool_choice {type: ${type}} to forced shim's function tool`, async () => { + test(`dispatcher rewrites tool_choice {type: ${type}} to forced dispatcher's function tool`, async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type }], @@ -550,17 +550,17 @@ for (const type of hostedAliasTypes) { }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); - assertEquals(inv.payload.tool_choice, { type: 'function', name: SHIM_TOOL_NAME }); + assertEquals(inv.payload.tool_choice, { type: 'function', name: FUNCTION_TOOL_NAME }); }); } // ── Per-tool fields propagate ────────────────────────────────────────── -test('shim propagates filters / user_location / search_context_size to backend', async () => { +test('dispatcher propagates filters / user_location / search_context_size to backend', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -578,7 +578,7 @@ test('shim propagates filters / user_location / search_context_size to backend', messageTurn('summary', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 1); const searchCall = backend.calls[0].request as WebSearchProviderRequest; @@ -591,13 +591,13 @@ test('shim propagates filters / user_location / search_context_size to backend', // ── Single-iteration message (no function calls) ─────────────────────── -test('shim forwards a single-turn message without backend calls', async () => { +test('dispatcher forwards a single-turn message without backend calls', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([messageTurn('hello back')]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(script.callCount(), 1); assertEquals(backend.calls.length, 0); @@ -612,16 +612,16 @@ test('shim forwards a single-turn message without backend calls', async () => { // ── One search then message ──────────────────────────────────────────── -test('shim drives one search then a final message in two upstream turns', async () => { +test('dispatcher drives one search then a final message in two upstream turns', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('summary', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const frames = await collectFrames(result.events); @@ -633,7 +633,7 @@ test('shim drives one search then a final message in two upstream turns', async assertEquals(tail[0].type, 'function_call'); assertEquals(tail[1].type, 'function_call_output'); assert(tail[0].type === 'function_call'); - // Shim replay preserves the upstream's original shim call + // Dispatcher replay preserves the upstream's original dispatcher call // verbatim (call_id, name, jsonrepair-canonical args) so the upstream // model on turn 2 sees its prior assistant turn unchanged. assertEquals(tail[0].call_id, 'call_1'); @@ -645,10 +645,10 @@ test('shim drives one search then a final message in two upstream turns', async test('translated source with a no-backing store still replays private search state across turns', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // A non-OpenAI-Responses source (Anthropic Messages/Gemini generateContent/OpenAI Chat Completions) translated into an OpenAI Responses - // upstream carries a no-backing store, not a persisting one; the shim's + // upstream carries a no-backing store, not a persisting one; the dispatcher's // request-private replay must still round-trip within the single request. const ctx = mockChatGatewayCtx({ apiKeyId: 'k1', wantsStream: true, store: createNonOpenAIResponsesSourceStore('k1') }); const script = scriptedRun([ @@ -656,7 +656,7 @@ test('translated source with a no-backing store still replays private search sta messageTurn('summary', 0), ]); - const result = await shim(inv, ctx, script.run); + const result = await dispatcher(inv, ctx, script.run); assert(result.type === 'events'); await collectFrames(result.events); @@ -672,7 +672,7 @@ test('translated source with a no-backing store still replays private search sta test('synthesized web_search_call ids retain request-private replay state', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const ctx = makeGatewayCtx(); const script = scriptedRun([ @@ -680,7 +680,7 @@ test('synthesized web_search_call ids retain request-private replay state', asyn messageTurn('summary', 0), ]); - const { frames } = await runShimAndDrain(shim, inv, ctx, script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, ctx, script.run); const doneEvents = outputItemDoneEvents(frames); const wsCallDoneIds = doneEvents.filter(e => e.item.type === 'web_search_call').map(e => e.item.id!); @@ -715,7 +715,7 @@ test('find_in_page reuses cache when same URL was opened first', async () => { }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ openCallTurn(0, 'call_o', 'https://example.com/p'), @@ -723,7 +723,7 @@ test('find_in_page reuses cache when same URL was opened first', async () => { messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); // open issues fetchPage; find reuses cache → still 1 fetchPage call. const fetchCalls = backend.calls.filter(c => c.kind === 'fetchPage'); @@ -734,14 +734,14 @@ test('find_in_page reuses cache when same URL was opened first', async () => { test('find_in_page triggers implicit fetchPage when URL not cached', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ findCallTurn(0, 'call_f', 'https://example.com/fresh', 'body'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const fetchCalls = backend.calls.filter(c => c.kind === 'fetchPage'); assertEquals(fetchCalls.length, 1); @@ -751,7 +751,7 @@ test('find_in_page triggers implicit fetchPage when URL not cached', async () => test('iteration cap returns the iteration-cap notice without backend call on cap+1', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const searchTurns: ScriptedTurn[] = []; for (let i = 0; i < 30; i++) { @@ -761,7 +761,7 @@ test('iteration cap returns the iteration-cap notice without backend call on cap searchTurns.push(messageTurn('done', 0)); const script = scriptedRun(searchTurns); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -777,7 +777,7 @@ test('iteration cap returns the iteration-cap notice without backend call on cap // lifecycle: 31 done events = 30 successful + 1 capped. The capped // call surfaces as the schema-error shape (action.type='search' with // the cap diagnostic in queries[0]) rather than the original query — - // the shim call is rejected before backend dispatch. + // the dispatcher call is rejected before backend dispatch. const wsCallDone = events.filter((e): e is Extract => e.type === 'response.output_item.done' && (e as { item?: { type?: string } }).item?.type === 'web_search_call'); @@ -789,25 +789,25 @@ test('iteration cap returns the iteration-cap notice without backend call on cap // ── Ambiguous multi-op function_call rejection ───────────────────────────── -// `web_search_call.action` carries exactly one action type; a shim call +// `web_search_call.action` carries exactly one action type; a dispatcher call // that mixes kinds (search + open) or stacks multiple non-search ops // (multi `open`/`find`) cannot reduce to a single wsc and is rejected // before backend dispatch. Multi-`search_query` collapses into one wsc // with a multi-query search action — see the collapse tests below. -const assertAmbiguousShimRejection = async (args: string, backend: { calls: unknown[] }): Promise => { - const shim = runHostedWebSearch; +const assertAmbiguousCallRejection = async (args: string, backend: { calls: unknown[] }): Promise => { + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_ambig', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_ambig', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, args), - mkFunctionCallDone(0, 'call_ambig', SHIM_TOOL_NAME, args), + mkFunctionCallDone(0, 'call_ambig', FUNCTION_TOOL_NAME, args), mkResponseCompleted(), ]; const script = scriptedRun([turn, messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); assertEquals(backend.calls.length, 0); @@ -822,25 +822,25 @@ const assertAmbiguousShimRejection = async (args: string, backend: { calls: unkn assert(item.results![0].snippet.includes('one web_search_call')); }; -test('multi-kind shim call (search_query + open) is rejected as ambiguous before any backend dispatch', async () => { +test('multi-kind dispatcher call (search_query + open) is rejected as ambiguous before any backend dispatch', async () => { const { backend } = makeStubDeps(); - await assertAmbiguousShimRejection( + await assertAmbiguousCallRejection( JSON.stringify({ search_query: [{ q: 'q' }], open: [{ ref_id: 'https://example.com/' }] }), backend, ); }); -test('multi-instance same-kind shim call (two open entries) is rejected as ambiguous', async () => { +test('multi-instance same-kind dispatcher call (two open entries) is rejected as ambiguous', async () => { const { backend } = makeStubDeps(); - await assertAmbiguousShimRejection( + await assertAmbiguousCallRejection( JSON.stringify({ open: [{ ref_id: 'https://example.com/a' }, { ref_id: 'https://example.com/b' }] }), backend, ); }); -test('multi-instance same-kind shim call (two find entries) is rejected as ambiguous', async () => { +test('multi-instance same-kind dispatcher call (two find entries) is rejected as ambiguous', async () => { const { backend } = makeStubDeps(); - await assertAmbiguousShimRejection( + await assertAmbiguousCallRejection( JSON.stringify({ find: [{ ref_id: 'https://example.com/', pattern: 'a' }, { ref_id: 'https://example.com/', pattern: 'b' }] }), backend, ); @@ -862,19 +862,19 @@ test('multi-`search_query` entries collapse into one web_search_call with a mult }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const args = JSON.stringify({ search_query: [{ q: 'q1' }, { q: 'q2' }, { q: 'q3' }] }); const turn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_multi', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_multi', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, args), - mkFunctionCallDone(0, 'call_multi', SHIM_TOOL_NAME, args), + mkFunctionCallDone(0, 'call_multi', FUNCTION_TOOL_NAME, args), mkResponseCompleted(), ]; const script = scriptedRun([turn, messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -899,7 +899,7 @@ test('multi-`search_query` entries collapse into one web_search_call with a mult test('multi-`search_query` with one malformed entry (missing q) is rejected as ambiguous', async () => { const { backend } = makeStubDeps(); - await assertAmbiguousShimRejection( + await assertAmbiguousCallRejection( JSON.stringify({ search_query: [{ q: 'q1' }, {}] }), backend, ); @@ -915,14 +915,14 @@ test('search returning zero results surfaces the "(no results)" template', async }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'empty-query'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -943,14 +943,14 @@ test('backend search failure surfaces "Search failed: "', async () => { }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -970,14 +970,14 @@ test('fetchPage whole-batch failure surfaces the open-page error text', async () }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ openCallTurn(0, 'call_o', 'https://example.com/x'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -990,14 +990,14 @@ test('fetchPage whole-batch failure surfaces the open-page error text', async () // ── Domain filter input validation ──────────────────────────────────── test('invalid request registration preserves an upstream error type and null code', async () => { - const shim = driveHostedToolStage([() => ({ + const dispatcher = driveHostedToolStage([() => ({ type: 'invalid-request', message: 'native image error', param: 'input', errorType: 'image_generation_user_error', code: null, })]); - const result = await shim( + const result = await dispatcher( makeInvocation(), makeGatewayCtx(), () => Promise.reject(new Error('run should not be called')), @@ -1049,9 +1049,9 @@ test('later hosted validation keeps original paths before another tool rewrites }, }); const originalInput = structuredClone(inv.payload.input); - const shim = driveHostedToolStage([webSearchHostedTool, imageGenerationHostedTool]); + const dispatcher = driveHostedToolStage([webSearchHostedTool, imageGenerationHostedTool]); - const result = await shim(inv, makeGatewayCtx(), async () => { + const result = await dispatcher(inv, makeGatewayCtx(), async () => { throw new Error('Invalid request reached upstream'); }); @@ -1067,7 +1067,7 @@ test('non-empty allowed_domains with every entry malformed is rejected as 400 in // loud 400 because the client believed they had a restrictive // allow-list. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1080,7 +1080,7 @@ test('non-empty allowed_domains with every entry malformed is rejected as 400 in }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'api-error'); assert(result.type === 'api-error'); assertEquals(result.status, 400); @@ -1099,7 +1099,7 @@ test('non-empty blocked_domains with every entry malformed is rejected as 400 in // every malformed entry would turn "block these sites" into "block // nothing", letting traffic the client intended to block through. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1112,7 +1112,7 @@ test('non-empty blocked_domains with every entry malformed is rejected as 400 in }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'api-error'); assert(result.type === 'api-error'); assertEquals(result.status, 400); @@ -1129,7 +1129,7 @@ test('domain lists with any malformed entry alongside valid entries are rejected // believed was blocked / outside the allow-list through. The // diagnostic must name the offending index so the client can fix it. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1145,7 +1145,7 @@ test('domain lists with any malformed entry alongside valid entries are rejected }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1157,12 +1157,12 @@ test('domain lists with any malformed entry alongside valid entries are rejected }); test('multiple hosted web_search entries: filter CONTENT is validated on each (not just last-wins)', async () => { - // `rewriteToolsForShim` last-wins on filter extraction, so a content + // `rewriteHostedTools` last-wins on filter extraction, so a content // check against `rewritten.filters` alone would miss malformed // entries on earlier hosted tools that get discarded. Per-entry // validation catches them — first failure wins. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1175,7 +1175,7 @@ test('multiple hosted web_search entries: filter CONTENT is validated on each (n }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1187,14 +1187,14 @@ test('multiple hosted web_search entries: filter CONTENT is validated on each (n test('omitted filters do not trigger the validation reject (no false positive on the default case)', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search' }], }, }); const script = scriptedRun([messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); await collectFrames(result.events); }); @@ -1206,7 +1206,7 @@ test('allowed_domains containing a non-string entry rejects with 400 (no 502 cra // 400 invalid_request_error keeps clients on the diagnostic-rich error // shape SDKs already speak. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1220,7 +1220,7 @@ test('allowed_domains containing a non-string entry rejects with 400 (no 502 cra }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1232,7 +1232,7 @@ test('allowed_domains containing a non-string entry rejects with 400 (no 502 cra test('blocked_domains containing an object entry rejects with 400', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1244,7 +1244,7 @@ test('blocked_domains containing an object entry rejects with 400', async () => }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1258,17 +1258,17 @@ test('blocked_domains containing an object entry rejects with 400', async () => // Each entry must be a bare hostname per OpenAI's web_search docs // (https://developers.openai.com/api/docs/guides/tools-web-search.md): // "omit the HTTP or HTTPS prefix" and use a domain like `openai.com`. -// The shim rejects every deviation at the boundary so silent surface +// The dispatcher rejects every deviation at the boundary so silent surface // expansion is impossible — `blocked_domains: ['reddit.com', // 'https://quora.com/']` used to silently leave quora unblocked // because the second entry dropped during normalization. -const runShimWithDomainEntry = async ( +const runWithDomainEntry = async ( field: 'allowed_domains' | 'blocked_domains', raw: unknown, ) => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1280,13 +1280,13 @@ const runShimWithDomainEntry = async ( }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); return { result, backend, script }; }; for (const field of ['allowed_domains', 'blocked_domains'] as const) { test(`${field} entry with https:// prefix rejects with 400 invalid domain`, async () => { - const { result, backend, script } = await runShimWithDomainEntry(field, 'https://quora.com/'); + const { result, backend, script } = await runWithDomainEntry(field, 'https://quora.com/'); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1298,7 +1298,7 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} entry with http:// prefix rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, 'http://example.com'); + const { result } = await runWithDomainEntry(field, 'http://example.com'); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1307,7 +1307,7 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} entry with a path rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, 'example.com/some/path'); + const { result } = await runWithDomainEntry(field, 'example.com/some/path'); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1316,7 +1316,7 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} entry with a port rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, 'example.com:8080'); + const { result } = await runWithDomainEntry(field, 'example.com:8080'); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1324,13 +1324,13 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} entry with a query string rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, 'example.com?q=1'); + const { result } = await runWithDomainEntry(field, 'example.com?q=1'); assert(result.type === 'api-error'); assertEquals(result.status, 400); }); test(`${field} empty-string entry rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, ''); + const { result } = await runWithDomainEntry(field, ''); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1338,13 +1338,13 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} whitespace-only entry rejects with 400 (no surface expansion via .trim())`, async () => { - const { result } = await runShimWithDomainEntry(field, ' '); + const { result } = await runWithDomainEntry(field, ' '); assert(result.type === 'api-error'); assertEquals(result.status, 400); }); test(`${field} non-string entry rejects with 400 (typed-but-not-string)`, async () => { - const { result } = await runShimWithDomainEntry(field, null); + const { result } = await runWithDomainEntry(field, null); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1357,14 +1357,14 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { test(`${field} exceeding 100 entries rejects with 400 (matches OpenAI documented cap)`, async () => { const oversized = Array.from({ length: 101 }, (_, i) => `host${i}.example`); const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { [field]: oversized } }], }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1379,14 +1379,14 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { test(`${field} exactly 100 entries is accepted (boundary inclusive)`, async () => { const exactly100 = Array.from({ length: 100 }, (_, i) => `host${i}.example`); makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { [field]: exactly100 } }], }, }); const script = scriptedRun([messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); await collectFrames(result.events); }); @@ -1397,7 +1397,7 @@ test('mixed list with one prefixed entry names the offending index (no silent dr // ['reddit.com', 'https://quora.com/'] previously had `quora.com` // silently NOT blocked. Reject at the boundary so the violating // index is named. - const { result } = await runShimMixedList('blocked_domains', ['reddit.com', 'https://quora.com/']); + const { result } = await runWithMixedList('blocked_domains', ['reddit.com', 'https://quora.com/']); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1405,12 +1405,12 @@ test('mixed list with one prefixed entry names the offending index (no silent dr assert(body.error.message.includes('quora.com')); }); -const runShimMixedList = async ( +const runWithMixedList = async ( field: 'allowed_domains' | 'blocked_domains', entries: unknown[], ) => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1422,7 +1422,7 @@ const runShimMixedList = async ( }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); return { result, backend, script }; }; @@ -1433,7 +1433,7 @@ test('invalid search_context_size value rejects with 400 (no silent fall-through // default — silently shrinking the result set. Reject at the // boundary so the misuse surfaces. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1442,7 +1442,7 @@ test('invalid search_context_size value rejects with 400 (no silent fall-through }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1454,14 +1454,14 @@ test('invalid search_context_size value rejects with 400 (no silent fall-through }); for (const field of ['external_web_access', 'search_content_types', 'return_token_budget'] as const) { - test(`explicitly-set hosted ${field} is silently stripped (the function tool the shim forwards never carries it)`, async () => { - // The shim replaces the hosted entry with its function tool; any - // hosted-only field — including ones the shim has no opinion on — + test(`explicitly-set hosted ${field} is silently stripped (the function tool the dispatcher forwards never carries it)`, async () => { + // The dispatcher replaces the hosted entry with its function tool; any + // hosted-only field — including ones the dispatcher has no opinion on — // drops out with the entry. Mirrors native: silently stripped. // Tests the request completes normally instead of being rejected // as a 400. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const fieldValue: Record = { external_web_access: false, search_content_types: ['image'], @@ -1473,7 +1473,7 @@ for (const field of ['external_web_access', 'search_content_types', 'return_toke }, }); const script = scriptedRun([messageTurn('hello', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); assertEquals(events[events.length - 1].type, 'response.completed'); @@ -1488,7 +1488,7 @@ test('array-shaped filters rejects with 400 (typeof null/[] === "object" guards // arrays at the boundary so the client sees the misuse instead of // a downstream filter behaving as if nothing was configured. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1500,7 +1500,7 @@ test('array-shaped filters rejects with 400 (typeof null/[] === "object" guards }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1518,14 +1518,14 @@ test('empty allowed_domains array is a no-op (not a misuse signal)', async () => // client input. Only a NON-empty list with all-malformed entries is // a misuse worth rejecting. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { allowed_domains: [] } }], }, }); const script = scriptedRun([messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); await collectFrames(result.events); }); @@ -1534,7 +1534,7 @@ test('null allowed_domains is a no-op (treated the same as omitted)', async () = // Tool authors sometimes use `null` and `undefined` interchangeably // as the "field absent" signal; neither should reject. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1546,7 +1546,7 @@ test('null allowed_domains is a no-op (treated the same as omitted)', async () = }, }); const script = scriptedRun([messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); await collectFrames(result.events); }); @@ -1555,7 +1555,7 @@ test('null allowed_domains is a no-op (treated the same as omitted)', async () = test('open blocked by domain filter never calls backend', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { blocked_domains: ['blocked.com'] } }], @@ -1566,7 +1566,7 @@ test('open blocked by domain filter never calls backend', async () => { messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 0); const input = inv.payload.input as OpenAIResponsesInputItem[]; @@ -1585,7 +1585,7 @@ test('find blocked by domain filter emits wire-side find_in_page (not open_page) // model's intent (e.g. a UI that draws a "searched inside // for " badge for find ops only). const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { blocked_domains: ['blocked.com'] } }], @@ -1596,7 +1596,7 @@ test('find blocked by domain filter emits wire-side find_in_page (not open_page) messageTurn('done', 0), ]); - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 0); // Wire-side: the synthesized output_item.done for the web_search_call @@ -1634,14 +1634,14 @@ test('find with no matches returns the no-matches text', async () => { }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ findCallTurn(0, 'call_f', 'https://example.com/p', 'nonexistent'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -1670,14 +1670,14 @@ test('find with matches returns bracketed context', async () => { }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ findCallTurn(0, 'call_f', 'https://example.com/p', 'needle'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -1707,14 +1707,14 @@ test('truncated page contents append the truncation sentinel', async () => { }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ openCallTurn(0, 'call_o', 'https://example.com/x'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -1729,7 +1729,7 @@ test('truncated page contents append the truncation sentinel', async () => { test('multi-turn merge: output_index unique, sequence_number monotonic, response.created once', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), @@ -1737,7 +1737,7 @@ test('multi-turn merge: output_index unique, sequence_number monotonic, response messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -1771,7 +1771,7 @@ test('multi-turn merge: output_index unique, sequence_number monotonic, response // Wire mock for upstream `response.created` that announces a specific // served model — distinct from the client's payload.model so we can -// prove the shim quotes upstream's served identity rather than the +// prove the dispatcher quotes upstream's served identity rather than the // requested literal. const mkResponseCreatedWithModel = (model: string, responseId = 'upstream_test'): ProtocolFrame => eventFrame({ @@ -1800,16 +1800,16 @@ test('synthesized response.created / completed quote the upstream-reported model // 'gpt-5.4-2025-01-20'. Every synthesized frame must mirror upstream. const SERVED_MODEL = 'gpt-5.4-2025-01-20'; makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { model: 'gpt-5' } }); const script = scriptedRun([ [ mkResponseCreatedWithModel(SERVED_MODEL), mkResponseInProgressWithModel(SERVED_MODEL), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, JSON.stringify({ search_query: [{ q: 'hi' }] })), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'hi' }] })), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'hi' }] })), mkResponseCompletedWithModel(SERVED_MODEL), ], [ @@ -1821,7 +1821,7 @@ test('synthesized response.created / completed quote the upstream-reported model ], ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -1836,7 +1836,7 @@ test('synthesized response.created / completed quote the upstream-reported model test('synthesized response.failed (upstream error mid-stream) quotes the upstream-reported model', async () => { const SERVED_MODEL = 'gpt-5.4-2025-01-20'; makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { model: 'gpt-5' } }); let runCalls = 0; @@ -1846,9 +1846,9 @@ test('synthesized response.failed (upstream error mid-stream) quotes the upstrea const frames: ProtocolFrame[] = [ mkResponseCreatedWithModel(SERVED_MODEL), mkResponseInProgressWithModel(SERVED_MODEL), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, JSON.stringify({ search_query: [{ q: 'hi' }] })), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'hi' }] })), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'hi' }] })), mkResponseCompletedWithModel(SERVED_MODEL), ]; const iterable: AsyncIterable> = (async function* () { @@ -1865,7 +1865,7 @@ test('synthesized response.failed (upstream error mid-stream) quotes the upstrea }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; @@ -1873,7 +1873,7 @@ test('synthesized response.failed (upstream error mid-stream) quotes the upstrea assertEquals(failed.response.model, SERVED_MODEL); }); -test('shim refuses to synthesize a response envelope when upstream response.created has no model', async () => { +test('dispatcher refuses to synthesize a response envelope when upstream response.created has no model', async () => { // No-fallback contract: a missing model on upstream's first // response.created is a protocol violation. The `ensureModel()` // invariant throws from inside `consumeTurnStreaming`; the throw @@ -1881,7 +1881,7 @@ test('shim refuses to synthesize a response envelope when upstream response.crea // (which reports it upward) rather than being silently swallowed // or papered over with `ctx.payload.model`. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { model: 'gpt-5' } }); const modelless = eventFrame({ @@ -1890,7 +1890,7 @@ test('shim refuses to synthesize a response envelope when upstream response.crea }); const script = scriptedRun([[modelless, mkResponseCompleted()]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -1904,13 +1904,13 @@ test('shim refuses to synthesize a response envelope when upstream response.crea assert(/did not report a `model`|never reported a `model`/.test(thrown.message)); }); -test('upstream response.created with no `id` field is tolerated (downstream uses the shim-synthesized id regardless)', async () => { - // The shim never quotes upstream's `id` downstream — every - // synthesized envelope carries the shim's own `resp_shim_` +test('upstream response.created with no `id` field is tolerated (downstream uses the gateway-synthesized id regardless)', async () => { + // The dispatcher never quotes upstream's `id` downstream — every + // synthesized envelope carries the dispatcher's own `resp_hosted_` // identity. So a missing id on upstream's `response.created` is // harmless; the downstream response still gets a valid id. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { model: 'gpt-5' } }); const idless = eventFrame({ @@ -1919,22 +1919,22 @@ test('upstream response.created with no `id` field is tolerated (downstream uses }); const script = scriptedRun([[idless, mkResponseCompleted()]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find(e => e.type === 'response.created') as Extract; - assert(created.response.id.startsWith('resp_shim_')); + assert(created.response.id.startsWith('resp_hosted_')); }); test('synthesized response.created / completed quote the once-per-request synthesized id (not the upstream id, not a fresh per-event id)', async () => { - // Upstream's id is irrelevant downstream — the shim generates a - // single `resp_shim_` at activation and quotes it on every + // Upstream's id is irrelevant downstream — the dispatcher generates a + // single `resp_hosted_` at activation and quotes it on every // synthesized envelope (created, in_progress, completed). One - // stable id is what clients correlate against across the shim's + // stable id is what clients correlate against across the dispatcher's // multi-turn response. const UPSTREAM_ID = 'resp_abc123def456'; makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ [ @@ -1946,14 +1946,14 @@ test('synthesized response.created / completed quote the once-per-request synthe ], ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find(e => e.type === 'response.created') as Extract; const inProgress = events.find(e => e.type === 'response.in_progress') as Extract; const completed = events.find(e => e.type === 'response.completed') as Extract; - assert(created.response.id.startsWith('resp_shim_')); + assert(created.response.id.startsWith('resp_hosted_')); assertEquals(inProgress.response.id, created.response.id); assertEquals(completed.response.id, created.response.id); // Upstream's id is not what the wire carries. @@ -1963,21 +1963,21 @@ test('synthesized response.created / completed quote the once-per-request synthe test('synthesized terminal id stays constant across multi-turn upstream id rotation', async () => { // Upstream re-encrypts response.id per turn (turn-N's // response.created and terminal frames can carry a different id - // than turn-1's). The shim's synthesized id is generated ONCE per + // than turn-1's). The dispatcher's synthesized id is generated ONCE per // request, so cross-turn synthesis quotes the same value end to // end regardless of what upstream rotates to. const TURN1_ID = 'resp_turn1_aaa'; const TURN2_ID = 'resp_turn2_bbb_rotated'; makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ [ mkResponseCreated(TURN1_ID), mkResponseInProgress(TURN1_ID), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, JSON.stringify({ search_query: [{ q: 'q' }] })), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'q' }] })), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'q' }] })), mkResponseCompleted(undefined, TURN1_ID), ], [ @@ -1989,13 +1989,13 @@ test('synthesized terminal id stays constant across multi-turn upstream id rotat ], ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find(e => e.type === 'response.created') as Extract; const completed = events.find(e => e.type === 'response.completed') as Extract; - assert(created.response.id.startsWith('resp_shim_')); - // Same shim-synthesized id end-to-end; neither upstream turn id + assert(created.response.id.startsWith('resp_hosted_')); + // Same gateway-synthesized id end-to-end; neither upstream turn id // leaks downstream. assertEquals(completed.response.id, created.response.id); assertFalse(completed.response.id === TURN1_ID); @@ -2006,7 +2006,7 @@ test('synthesized terminal id stays constant across multi-turn upstream id rotat test('usage accumulates across three iterations', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turnWithUsage = ( @@ -2018,9 +2018,9 @@ test('usage accumulates across three iterations', async () => { return [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, callId, SHIM_TOOL_NAME), + mkFunctionCallAdded(0, callId, FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, args), - mkFunctionCallDone(0, callId, SHIM_TOOL_NAME, args), + mkFunctionCallDone(0, callId, FUNCTION_TOOL_NAME, args), mkResponseCompleted({ input_tokens: inTok, output_tokens: outTok, total_tokens: inTok + outTok }), ]; }; @@ -2039,7 +2039,7 @@ test('usage accumulates across three iterations', async () => { finalTurn, ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find((e): e is Extract => e.type === 'response.completed'); @@ -2055,16 +2055,16 @@ test('usage cached_tokens reported on one turn carries through (last-turn omissi // turn 2 omits it. Final usage must still surface cached_tokens // (turn 1's value). makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn1Args = JSON.stringify({ search_query: [{ q: 'q1' }] }); const turn1: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, turn1Args), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, turn1Args), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, turn1Args), mkResponseCompleted({ input_tokens: 100, output_tokens: 50, @@ -2081,7 +2081,7 @@ test('usage cached_tokens reported on one turn carries through (last-turn omissi mkResponseCompleted({ input_tokens: 50, output_tokens: 10, total_tokens: 60 }), ]; - const result = await shim(inv, makeGatewayCtx(), scriptedRun([turn1, turn2]).run); + const result = await dispatcher(inv, makeGatewayCtx(), scriptedRun([turn1, turn2]).run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find((e): e is Extract => e.type === 'response.completed'); @@ -2099,16 +2099,16 @@ test('usage cached_tokens never reported on any turn is omitted from wire (no fa // as `cached_tokens: 0`. The wire shape matches what a native // upstream that doesn't track caching would produce. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn1Args = JSON.stringify({ search_query: [{ q: 'q1' }] }); const turn1: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, turn1Args), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, turn1Args), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, turn1Args), mkResponseCompleted({ input_tokens: 100, output_tokens: 50, total_tokens: 150 }), ]; const turn2: ScriptedTurn = [ @@ -2119,7 +2119,7 @@ test('usage cached_tokens never reported on any turn is omitted from wire (no fa mkResponseCompleted({ input_tokens: 50, output_tokens: 10, total_tokens: 60 }), ]; - const result = await shim(inv, makeGatewayCtx(), scriptedRun([turn1, turn2]).run); + const result = await dispatcher(inv, makeGatewayCtx(), scriptedRun([turn1, turn2]).run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find((e): e is Extract => e.type === 'response.completed'); @@ -2129,7 +2129,7 @@ test('usage cached_tokens never reported on any turn is omitted from wire (no fa assertEquals(completed.response.usage?.output_tokens_details, undefined); }); -test('next-turn function_call echo always carries the canonical re-stringified shim call args (single shape unified with client-roundtrip)', async () => { +test('next-turn function_call echo always carries the canonical re-stringified dispatcher call args (single shape unified with client-roundtrip)', async () => { // The dispatcher always overwrites the intercepted call's // arguments with the canonical re-stringified form, regardless of // whether the upstream string was already valid JSON. This @@ -2142,21 +2142,21 @@ test('next-turn function_call echo always carries the canonical re-stringified s // upstream would 400 on the broken raw string — but the same // canonical form is also used for already-valid input. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Trailing-comma JSON survives jsonrepair but is not valid strict JSON. const brokenArgs = '{"search_query":[{"q":"q"}],}'; const brokenTurn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_broken', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_broken', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, brokenArgs), - mkFunctionCallDone(0, 'call_broken', SHIM_TOOL_NAME, brokenArgs), + mkFunctionCallDone(0, 'call_broken', FUNCTION_TOOL_NAME, brokenArgs), mkResponseCompleted(), ]; const script = scriptedRun([brokenTurn, messageTurn('done', 0)]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const fc = input.find(i => i.type === 'function_call') as @@ -2168,7 +2168,7 @@ test('next-turn function_call echo always carries the canonical re-stringified s assertEquals(fc.arguments, '{"search_query":[{"q":"q"}]}'); }); -test('upstream sends bare `error` frame BEFORE any response.created: shim throws from the events iterator (no synthetic response.failed with empty identity)', async () => { +test('upstream sends bare `error` frame BEFORE any response.created: dispatcher throws from the events iterator (no synthetic response.failed with empty identity)', async () => { // A bare `{type:'error'}` arriving before any `response.created` // produces no captured identity (id / model). Synthesizing a // `response.failed` envelope would require those fields. The @@ -2178,7 +2178,7 @@ test('upstream sends bare `error` frame BEFORE any response.created: shim throws // honest about the upstream protocol violation rather than // silently lying about the served identity. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const errorOnlyTurn: ScriptedTurn = [ eventFrame({ @@ -2188,7 +2188,7 @@ test('upstream sends bare `error` frame BEFORE any response.created: shim throws ]; const script = scriptedRun([errorOnlyTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -2199,13 +2199,13 @@ test('upstream sends bare `error` frame BEFORE any response.created: shim throws assert(thrown instanceof Error); }); -test('upstream sends bare `error` frame AFTER response.created: shim emits response.failed with synthesized id + last-seen model', async () => { - // When model is captured before the bare error arrives, the shim +test('upstream sends bare `error` frame AFTER response.created: dispatcher emits response.failed with synthesized id + last-seen model', async () => { + // When model is captured before the bare error arrives, the dispatcher // can synthesize a wire-valid `response.failed` envelope. The - // synthesized id is the shim's own (`resp_shim_`), and the + // synthesized id is the dispatcher's own (`resp_hosted_`), and the // model is the last-seen upstream model. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const createdWith = eventFrame({ type: 'response.created', @@ -2228,12 +2228,12 @@ test('upstream sends bare `error` frame AFTER response.created: shim emits respo }), ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; assertEquals(failed.type, 'response.failed'); - assert(failed.response.id.startsWith('resp_shim_')); + assert(failed.response.id.startsWith('resp_hosted_')); assertEquals(failed.response.model, 'gpt-5.4-2025-01-20'); assertEquals(failed.response.status, 'failed'); assertEquals(failed.response.error?.message, 'mid-stream upstream blew up'); @@ -2245,7 +2245,7 @@ test('upstream sends bare `error` frame AFTER response.created: shim emits respo assertFalse('type' in (failed.response.error as object)); }); -test('upstream iterator rejects before yielding any frame: shim surfaces the throw through the events iterator', async () => { +test('upstream iterator rejects before yielding any frame: dispatcher surfaces the throw through the events iterator', async () => { // An iterator that throws synchronously on `.next()` (malformed // SSE JSON, reader rejection, network reset mid-handshake) yields // no identity capture. The synthesizer can't build a wire-valid @@ -2253,7 +2253,7 @@ test('upstream iterator rejects before yielding any frame: shim surfaces the thr // the source responder reports it upward — same channel as any // other mid-stream failure with no captured identity. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const failingIterator: AsyncIterable> = (async function* () { throw new Error('malformed SSE JSON at byte 42'); @@ -2265,7 +2265,7 @@ test('upstream iterator rejects before yielding any frame: shim surfaces the thr modelIdentity: testTelemetryModelIdentity, }); - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -2276,7 +2276,7 @@ test('upstream iterator rejects before yielding any frame: shim surfaces the thr assert(thrown instanceof Error); }); -test('pathological upstream emitting frames without response.created: shim eventually completes without identity capture (no infinite-loop)', async () => { +test('pathological upstream emitting frames without response.created: dispatcher eventually completes without identity capture (no infinite-loop)', async () => { // The previous preflight enforced a 100-frame budget here. With // preflight removed, indexed-unknown frames flow through // consume-turn directly. The protective behavior now relies on @@ -2287,7 +2287,7 @@ test('pathological upstream emitting frames without response.created: shim event // (matches a malformed-but-finite upstream that drops without // ever sending a shell). makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const noisyIterator: AsyncIterable> = (async function* () { for (let i = 0; i < 5; i++) { @@ -2304,7 +2304,7 @@ test('pathological upstream emitting frames without response.created: shim event modelIdentity: testTelemetryModelIdentity, }); - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -2321,7 +2321,7 @@ test('turn-1 iterator throws AFTER response.created: synthesizes response.failed // downstream wire mirrors a native upstream's mid-stream drop // instead of escaping uncaught to a gateway internal_error. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const failingMidStream: AsyncIterable> = (async function* () { yield mkResponseCreated('upstream_mid'); @@ -2333,14 +2333,14 @@ test('turn-1 iterator throws AFTER response.created: synthesizes response.failed modelIdentity: testTelemetryModelIdentity, }); - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const terminal = events[events.length - 1]; assertEquals(terminal.type, 'response.failed'); const failed = terminal as Extract; assertEquals(failed.response.status, 'failed'); - assert(failed.response.id.startsWith('resp_shim_')); + assert(failed.response.id.startsWith('resp_hosted_')); assertEquals(failed.response.model, 'test-model'); assertEquals(failed.response.error?.code, 'server_error'); assert(failed.response.error?.message.includes('connection reset by peer')); @@ -2355,7 +2355,7 @@ test('turn-2 iterator throws: synthesizes response.failed with captured id/model // A throw after the multi-turn loop has crossed run() boundaries // must still funnel through the `response.failed` synthesizer. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -2375,7 +2375,7 @@ test('turn-2 iterator throws: synthesizes response.failed with captured id/model return { type: 'events', events: failingTurn2, modelIdentity: testTelemetryModelIdentity }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const terminal = events[events.length - 1]; @@ -2387,15 +2387,15 @@ test('turn-2 iterator throws: synthesizes response.failed with captured id/model assertFalse('type' in (failed.response.error as object)); }); -test('consume-turn finishes without identity AND without bare-error-pre-shell: shim throws from synthesizer (no silent invented identity)', async () => { +test('consume-turn finishes without identity AND without bare-error-pre-shell: dispatcher throws from synthesizer (no silent invented identity)', async () => { // A `response.completed` arriving without a preceding // `response.created` reaches the terminal completed state without // capturing identity. Synthesizing a completed envelope would - // require id + model the shim never observed — `ensureModel` / + // require id + model the dispatcher never observed — `ensureModel` / // `ensureResponseId` throw instead of inventing values, and the // throw escapes the events generator. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const malformed: AsyncIterable> = (async function* () { yield mkResponseCompleted(); @@ -2406,7 +2406,7 @@ test('consume-turn finishes without identity AND without bare-error-pre-shell: s modelIdentity: testTelemetryModelIdentity, }); - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -2420,7 +2420,7 @@ test('consume-turn finishes without identity AND without bare-error-pre-shell: s // ── Snapshot pass-through preserves upstream-owned envelope fields ── test('snapshot pass-through: upstream tools/tool_choice/temperature/parallel_tool_calls/reasoning/service_tier survive synthesized response.created + .completed', async () => { - // The shim used to build fresh OpenAIResponsesResult shells with hard- + // The dispatcher used to build fresh OpenAIResponsesResult shells with hard- // coded {id, object, model, status, output, output_text, [usage]} // and drop everything else upstream sent. Real upstream wire frames // carry parallel_tool_calls, tool_choice, tools, temperature, top_p, @@ -2465,11 +2465,11 @@ test('snapshot pass-through: upstream tools/tool_choice/temperature/parallel_too } as OpenAIResponsesResult, }); makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([[createdWith, mkMessageAdded(0), mkMessageDone(0, 'done'), completedWith]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find(e => e.type === 'response.created') as Extract; @@ -2499,7 +2499,7 @@ test('snapshot pass-through: upstream tools/tool_choice/temperature/parallel_too test('snapshot pass-through: synthesized response.failed (mid-stream upstream error) preserves upstream-owned envelope fields from turn 1 snapshot', async () => { // The error path builds its `response.failed` envelope from the - // shim's accumulated state but still spreads the captured snapshot + // dispatcher's accumulated state but still spreads the captured snapshot // so client tool_choice / tools / reasoning visibility survives even // when a later turn fails. const upstreamSnapshot = { @@ -2518,7 +2518,7 @@ test('snapshot pass-through: synthesized response.failed (mid-stream upstream er response: upstreamSnapshot as OpenAIResponsesResult, }); makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -2528,9 +2528,9 @@ test('snapshot pass-through: synthesized response.failed (mid-stream upstream er const wsArgs = JSON.stringify({ search_query: [{ q: 'q' }] }); const frames: AsyncIterable> = (async function* () { yield createdWith; - yield mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME); + yield mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME); yield mkFunctionCallArgsDone(0, wsArgs); - yield mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, wsArgs); + yield mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, wsArgs); yield eventFrame({ type: 'response.completed', response: { ...upstreamSnapshot, status: 'completed' } as OpenAIResponsesResult, @@ -2547,7 +2547,7 @@ test('snapshot pass-through: synthesized response.failed (mid-stream upstream er }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events.find(e => e.type === 'response.failed') as Extract; @@ -2557,9 +2557,9 @@ test('snapshot pass-through: synthesized response.failed (mid-stream upstream er assertEquals(r.reasoning, { effort: 'high' }); }); -test('snapshot pass-through: snapshot fields like completed_at flow through verbatim onto every synthesized envelope (the shim only overrides id/model/status/output/usage)', async () => { +test('snapshot pass-through: snapshot fields like completed_at flow through verbatim onto every synthesized envelope (the dispatcher only overrides id/model/status/output/usage)', async () => { // Synthesizers spread upstream's snapshot in full and override only - // the shim-owned fields. Snapshot pass-through is the contract — + // the gateway-owned fields. Snapshot pass-through is the contract — // a turn-2 synthesizer reads turn-1's snapshot until refreshed by // a later upstream frame, so `completed_at`-style fields can ride // through onto a multi-turn failed envelope. Clients reading @@ -2588,7 +2588,7 @@ test('snapshot pass-through: snapshot fields like completed_at flow through verb } as unknown as OpenAIResponsesResult, }); makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -2598,9 +2598,9 @@ test('snapshot pass-through: snapshot fields like completed_at flow through verb const wsArgs = JSON.stringify({ search_query: [{ q: 'q' }] }); const frames: AsyncIterable> = (async function* () { yield turn1Created; - yield mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME); + yield mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME); yield mkFunctionCallArgsDone(0, wsArgs); - yield mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, wsArgs); + yield mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, wsArgs); yield turn1Completed; })(); return { type: 'events', events: frames, modelIdentity: testTelemetryModelIdentity }; @@ -2614,7 +2614,7 @@ test('snapshot pass-through: snapshot fields like completed_at flow through verb }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events.find(e => e.type === 'response.failed') as Extract; @@ -2653,7 +2653,7 @@ test('snapshot strip: emitFinalCompleted re-adds completed_at when upstream supp } as unknown as OpenAIResponsesResult, }); makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([[ turn1Created, @@ -2662,7 +2662,7 @@ test('snapshot strip: emitFinalCompleted re-adds completed_at when upstream supp turn1Completed, ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find(e => e.type === 'response.completed') as Extract; @@ -2674,7 +2674,7 @@ test('snapshot strip: emitFinalCompleted re-adds completed_at when upstream supp test('snapshot pass-through: incomplete_details: null on the captured snapshot propagates verbatim to the synthesized response.completed', async () => { // Native upstreams sometimes emit `incomplete_details: null` on a // `response.completed` (Pydantic round-trips `None` for explicit - // null vs. omitted-field distinctions). The shim must preserve the + // null vs. omitted-field distinctions). The dispatcher must preserve the // exact field state — coercing `null` to omitted (or vice versa) // would break clients that probe for the field's presence rather // than its truthiness. @@ -2700,7 +2700,7 @@ test('snapshot pass-through: incomplete_details: null on the captured snapshot p } as unknown as OpenAIResponsesResult, }); makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([[ turn1Created, @@ -2709,7 +2709,7 @@ test('snapshot pass-through: incomplete_details: null on the captured snapshot p turn1Completed, ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find(e => e.type === 'response.completed') as Extract; @@ -2726,18 +2726,18 @@ test('success-path synth envelopes carry spec-required `error: null` and `incomp // REQUIRED on every Response (both nullable). Reference: // https://github.com/openai/openai-openapi/blob/master/openapi.yaml // `Response.required` lists both. Typed-SDK clients parse against - // that contract and reject envelopes missing the keys; the shim's + // that contract and reject envelopes missing the keys; the dispatcher's // `overlayOnSnapshot` therefore defaults both to null after stripping // terminal-only snapshot fields, then lets explicit overlays (e.g. // `response.failed.error`, real terminal `incomplete_details`) win. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Single turn, pure message, no tool call — exercises the // emitFinalCompleted success path through overlayOnSnapshot. const script = scriptedRun([messageTurn('hi')]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -2756,11 +2756,11 @@ test('success-path synth envelopes carry spec-required `error: null` and `incomp // ── output_text is never synthesized ───────────────────────────────── -test('shim rebuilds `output_text` on terminal envelopes from the accumulated message items (matches openai-python Response.output_text)', async () => { +test('dispatcher rebuilds `output_text` on terminal envelopes from the accumulated message items (matches openai-python Response.output_text)', async () => { // SDKs derive `output_text` from the `output` array. Per-turn // upstream `output_text` on a terminal frame only describes that - // one turn, so on multi-turn shim responses the snapshot value - // would desync from the cross-turn aggregated `output`. The shim + // one turn, so on multi-turn dispatcher responses the snapshot value + // would desync from the cross-turn aggregated `output`. The dispatcher // rebuilds the alias from `accumulatedOutput` and overrides the // snapshot's value on the terminal envelope. const omitOutputText = (responseId = 'upstream_test'): ProtocolFrame => @@ -2790,7 +2790,7 @@ test('shim rebuilds `output_text` on terminal envelopes from the accumulated mes }, }); makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn: ScriptedTurn = [ omitOutputTextCreated(), @@ -2800,7 +2800,7 @@ test('shim rebuilds `output_text` on terminal envelopes from the accumulated mes ]; const script = scriptedRun([turn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find((e): e is Extract => e.type === 'response.completed'); @@ -2810,7 +2810,7 @@ test('shim rebuilds `output_text` on terminal envelopes from the accumulated mes test('upstream-emitted `output_text` on in-progress envelopes flows through verbatim from the snapshot', async () => { // Snapshot pass-through contract: every upstream field flows - // through unchanged unless the shim explicitly overrides it. + // through unchanged unless the dispatcher explicitly overrides it. // `output_text` on in-progress envelopes is just the snapshot's // value — terminal envelopes get a separately-rebuilt // `output_text` aggregated across turns (covered by the dedicated @@ -2856,7 +2856,7 @@ test('upstream-emitted `output_text` on in-progress envelopes flows through verb } as OpenAIResponsesResult, }); makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([[ createdWithStaleText, @@ -2866,7 +2866,7 @@ test('upstream-emitted `output_text` on in-progress envelopes flows through verb completedFrame, ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find((e): e is Extract => e.type === 'response.created'); @@ -2874,7 +2874,7 @@ test('upstream-emitted `output_text` on in-progress envelopes flows through verb assert(created !== undefined); assert(inProgress !== undefined); // Snapshot's output_text flows through verbatim on in-progress - // envelopes — the shim only overrides id/model/status/output. + // envelopes — the dispatcher only overrides id/model/status/output. assertEquals((created.response as unknown as { output_text?: string }).output_text, 'snapshot output_text'); assertEquals((inProgress.response as unknown as { output_text?: string }).output_text, 'snapshot output_text'); }); @@ -2889,7 +2889,7 @@ test('finalMetadata resolves with the LATEST turn modelIdentity, not turn 1', as // through firstResult.finalMetadata would freeze it to turn 1's // modelKey. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -2915,7 +2915,7 @@ test('finalMetadata resolves with the LATEST turn modelIdentity, not turn 1', as }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); // Drain so the multi-turn loop completes and the reading it hands up settles. await collectFrames(result.events); @@ -2934,14 +2934,14 @@ test('synthesized web_search_call (search action) carries both `query` (singular // `queries: list[str]`. Emit BOTH so every typed SDK reads a // populated value regardless of which field its model declares. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'who invented graphql'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -2955,18 +2955,18 @@ test('synthesized web_search_call (search action) carries both `query` (singular test('response.output_item.added for web_search_call omits action (mirrors native — action populated only on .done)', async () => { // Native upstreams omit `action` on the `.added` half and populate - // it only on `.done` once the operation completes. The shim + // it only on `.done` once the operation completes. The dispatcher // follows suit; clients that render action.* read from the done // frame. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'who invented graphql'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const wsAdded = events.find( @@ -2996,21 +2996,21 @@ test('open with invalid ref_id: done frame carries action.type="search" with the // typed-SDK clients render a coherent intent). The added frame // omits `action` regardless, matching native. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const invalidRef = 'turn_4_ws_0'; const argsJson = JSON.stringify({ open: [{ ref_id: invalidRef }] }); const turn1: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_open', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_open', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, argsJson), - mkFunctionCallDone(0, 'call_open', SHIM_TOOL_NAME, argsJson), + mkFunctionCallDone(0, 'call_open', FUNCTION_TOOL_NAME, argsJson), mkResponseCompleted(), ]; const script = scriptedRun([turn1, messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const wsDone = events.find( @@ -3042,12 +3042,12 @@ test('open with valid URL whose fetch fails: done frame carries action.type="ope }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn1 = openCallTurn(0, 'call_open', failingUrl); const script = scriptedRun([turn1, messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const wsDone = events.find( @@ -3066,8 +3066,8 @@ test('open with valid URL whose fetch fails: done frame carries action.type="ope test('include: ["web_search_call.action.sources"] populates action.sources with search-result URLs', async () => { // Native OpenAI Responses gates `action.sources` on this exact include // token. Clients reading `web_search_call.action.sources` against - // a native upstream see the URLs of every search hit; the shim - // mirrors that opt-in shape so a switch from native to shim is + // a native upstream see the URLs of every search hit; the dispatcher + // mirrors that opt-in shape so a switch from native to dispatcher is // observably identical. makeStubDeps({ providerOverrides: { @@ -3082,7 +3082,7 @@ test('include: ["web_search_call.action.sources"] populates action.sources with }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { include: ['web_search_call.action.sources'] }, }); @@ -3091,7 +3091,7 @@ test('include: ["web_search_call.action.sources"] populates action.sources with messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -3106,14 +3106,14 @@ test('include: ["web_search_call.action.sources"] populates action.sources with test('without include: ["web_search_call.action.sources"], action.sources is absent (opt-in shape matches native)', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -3125,14 +3125,14 @@ test('without include: ["web_search_call.action.sources"], action.sources is abs test('web_search_call results field is populated on the wire when the client opted in via include: ["web_search_call.results"]', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Default already opts in. const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -3148,14 +3148,14 @@ test('web_search_call results field is omitted from the wire when the client did // therefore the persisted `payload.private`) still holds the full // results so a later-turn echo can be hydrated. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { include: [] } }); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -3164,23 +3164,23 @@ test('web_search_call results field is omitted from the wire when the client did assertEquals(item.results, undefined); }); -// ── Mixed-tool turn (shim call + client tool present) ───────────────── +// ── Mixed-tool turn (dispatcher call + client tool present) ───────────────── -test('mixed-tool: shim call + client function_call exits to client after one turn, with both sets of items downstream', async () => { - // GPT-5.x emits both kinds of tool calls in one turn. The shim executes - // the shim's searches server-side (so the client sees completed +test('mixed-tool: dispatcher call + client function_call exits to client after one turn, with both sets of items downstream', async () => { + // GPT-5.x emits both kinds of tool calls in one turn. The dispatcher executes + // the dispatcher's searches server-side (so the client sees completed // web_search_call lifecycles) and lets the client round-trip its own // function_call. No internal rerun, no rejection injection. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const wsArgs = JSON.stringify({ search_query: [{ q: 'q1' }] }); const mixedTurn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, wsArgs), - mkFunctionCallDone(0, 'call_ws', SHIM_TOOL_NAME, wsArgs), + mkFunctionCallDone(0, 'call_ws', FUNCTION_TOOL_NAME, wsArgs), mkFunctionCallAdded(1, 'call_other', 'lookup'), mkFunctionCallArgsDone(1, '{"q":"x"}', 'fc_1'), mkFunctionCallDone(1, 'call_other', 'lookup', '{"q":"x"}'), @@ -3189,7 +3189,7 @@ test('mixed-tool: shim call + client function_call exits to client after one tur const script = scriptedRun([mixedTurn]); const originalInputLen = (inv.payload.input as OpenAIResponsesInputItem[]).length; - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 1); assertEquals(script.callCount(), 1); @@ -3205,17 +3205,17 @@ test('mixed-tool: shim call + client function_call exits to client after one tur assertEquals(wsLifecycleCount, 3); }); -test('mixed-tool: shim call + custom_tool_call exits to client; custom_tool_call frames flush downstream', async () => { +test('mixed-tool: dispatcher call + custom_tool_call exits to client; custom_tool_call frames flush downstream', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const wsArgs = JSON.stringify({ search_query: [{ q: 'q' }] }); const mixedTurn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, wsArgs), - mkFunctionCallDone(0, 'call_ws', SHIM_TOOL_NAME, wsArgs), + mkFunctionCallDone(0, 'call_ws', FUNCTION_TOOL_NAME, wsArgs), mkCustomToolCallAdded(1, 'call_ct', 'my_freeform_tool'), mkCustomToolCallInputDone(1, 'raw input'), mkCustomToolCallDone(1, 'call_ct', 'my_freeform_tool', 'raw input'), @@ -3223,7 +3223,7 @@ test('mixed-tool: shim call + custom_tool_call exits to client; custom_tool_call ]; const script = scriptedRun([mixedTurn]); - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 1); assertEquals(script.callCount(), 1); @@ -3235,9 +3235,9 @@ test('mixed-tool: shim call + custom_tool_call exits to client; custom_tool_call assert(customAdded !== undefined); }); -test('client-only tool turn (no shim call): pass-through function_call frames flush downstream and close the loop', async () => { +test('client-only tool turn (no dispatcher call): pass-through function_call frames flush downstream and close the loop', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const clientOnlyTurn: ScriptedTurn = [ mkResponseCreated(), @@ -3250,7 +3250,7 @@ test('client-only tool turn (no shim call): pass-through function_call frames fl const script = scriptedRun([clientOnlyTurn]); const originalInputLen = (inv.payload.input as OpenAIResponsesInputItem[]).length; - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); // Loop closes after the single turn — the client drives the next round. assertEquals(script.callCount(), 1); assertEquals(backend.calls.length, 0); @@ -3266,20 +3266,20 @@ test('client-only tool turn (no shim call): pass-through function_call frames fl // ── Pass-through of replayed web_search_call from input ──────────────── -// ── Input preprocessor: web_search_call items → shim call pair ───────── +// ── Input preprocessor: web_search_call items → dispatcher call pair ───────── -test('input preprocessor: each web_search_call item becomes one shim call + function_call_output pair', async () => { - // Upstream knows the shim call only as a function tool; a hosted +test('input preprocessor: each web_search_call item becomes one dispatcher call + function_call_output pair', async () => { + // Upstream knows the dispatcher call only as a function tool; a hosted // `web_search_call` item type in its input would be unrecognized. We // translate each echoed item into a function_call + function_call_output // pair that the upstream model can reason over. With no per-item - // private payload (this test runs raw through the shim), the + // private payload (this test runs raw through the dispatcher), the // function_call mirrors the wire action shape and the - // function_call_output is the placeholder — the shim deliberately + // function_call_output is the placeholder — the dispatcher deliberately // ignores the wire `results` field because gateway-side state is the // only source of truth. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const replayedSearch: OpenAIResponsesInputWebSearchCall = { type: 'web_search_call', id: 'ws_old_search', @@ -3306,7 +3306,7 @@ test('input preprocessor: each web_search_call item becomes one shim call + func }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; // Original 2 messages + 2 expanded pairs (2 items each) = 6 items. @@ -3319,7 +3319,7 @@ test('input preprocessor: each web_search_call item becomes one shim call + func assertFalse(input.some(i => i.type === 'web_search_call')); // The pair from the search reflects its action. const searchFc = input[1] as { name: string; arguments: string }; - assertEquals(searchFc.name, SHIM_TOOL_NAME); + assertEquals(searchFc.name, FUNCTION_TOOL_NAME); assert(searchFc.arguments.includes('hello world')); // No payload → output is the not-preserved placeholder, not the // wire snippet. A re-search prompt is emitted when data is needed. @@ -3333,7 +3333,7 @@ test('input preprocessor: each web_search_call item becomes one shim call + func test('input preprocessor: replay-only activation leaves hosted tool_choice unchanged when no hosted tool is declared', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [], @@ -3352,7 +3352,7 @@ test('input preprocessor: replay-only activation leaves hosted tool_choice uncha }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tool_choice, { type: 'web_search_preview' }); const input = inv.payload.input as OpenAIResponsesInputItem[]; @@ -3364,12 +3364,12 @@ test('input preprocessor: web_search_call without an action is replaced by a pla // item entirely silently shortens conversation history, which can // mislead the model (e.g. its reasoning about "your last 3 tool // calls" no longer matches reality). Replace with a placeholder - // shim call (empty args, no logical ops) + a + // dispatcher call (empty args, no logical ops) + a // function_call_output telling the model the prior contents // weren't preserved. Same idea as the no-payload-with-action path // for partial echoes. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { input: [ @@ -3383,7 +3383,7 @@ test('input preprocessor: web_search_call without an action is replaced by a pla }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; // user message + placeholder function_call + placeholder function_call_output. @@ -3392,7 +3392,7 @@ test('input preprocessor: web_search_call without an action is replaced by a pla assertEquals(input[1].type, 'function_call'); assertEquals(input[2].type, 'function_call_output'); const fc = input[1] as { type: 'function_call'; name: string; arguments: string }; - assertEquals(fc.name, SHIM_TOOL_NAME); + assertEquals(fc.name, FUNCTION_TOOL_NAME); // Empty args → upstream model sees no logical operations, // matching the intent of "we don't know what this call did". assertEquals(fc.arguments, '{}'); @@ -3406,7 +3406,7 @@ test('input preprocessor: web_search_call without an action is replaced by a pla test('input preprocessor: web_search_call with empty id has its id synthesized and produces an upstream pair', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { input: [ @@ -3421,7 +3421,7 @@ test('input preprocessor: web_search_call with empty id has its id synthesized a }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; // user message + function_call + function_call_output (paired from the @@ -3437,7 +3437,7 @@ test('input preprocessor: web_search_call without results emits the not-preserve // synthesizing a phantom zero-hit response that would mislead the // model into thinking the prior search returned nothing. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { input: [ @@ -3451,7 +3451,7 @@ test('input preprocessor: web_search_call without results emits the not-preserve }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; assertEquals(input.length, 2); @@ -3459,11 +3459,11 @@ test('input preprocessor: web_search_call without results emits the not-preserve assertEquals(fco.output, 'Prior search results were not preserved in the conversation history. Call web_search again if you need them.'); }); -// ── Shim tool name resolution / collision fallback ──────────────── +// ── Dispatcher tool name resolution / collision fallback ──────────────── -test('client declaring web_search + hosted web_search: shim falls back to web_search_2', async () => { +test('client declaring web_search + hosted web_search: dispatcher falls back to web_search_2', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -3478,15 +3478,15 @@ test('client declaring web_search + hosted web_search: shim falls back to web_se messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const names = new Set((inv.payload.tools ?? []).map(t => (t as { name: string }).name)); assertEquals(names, new Set(['web_search_2', 'web_search'])); }); -test('client declaring web_search AND web_search_2 + hosted web_search: shim falls back to web_search_3', async () => { +test('client declaring web_search AND web_search_2 + hosted web_search: dispatcher falls back to web_search_3', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -3502,7 +3502,7 @@ test('client declaring web_search AND web_search_2 + hosted web_search: shim fal messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const names = new Set((inv.payload.tools ?? []).map(t => (t as { name: string }).name)); assertEquals(names, new Set(['web_search', 'web_search_2', 'web_search_3'])); @@ -3511,11 +3511,11 @@ test('client declaring web_search AND web_search_2 + hosted web_search: shim fal for (const [source, item] of [ ['additional_tools', { type: 'additional_tools', role: 'developer', - tools: [{ type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object' } }], + tools: [{ type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object' } }], }], ['tool_search_output', { type: 'tool_search_output', execution: 'client', call_id: 'call_search', - tools: [{ type: 'custom', name: SHIM_TOOL_NAME }], + tools: [{ type: 'custom', name: FUNCTION_TOOL_NAME }], }], ] as const) { test(`hosted function reserves a distinct alias beside a flat client tool from historical ${source}`, async () => { @@ -3527,12 +3527,12 @@ for (const [source, item] of [ }, }); const script = scriptedRun([messageTurn('done')]); - const { result } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + const { result } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'events'); assertEquals(script.callCount(), 1); const injected = inv.payload.tools?.[0]; assert(injected?.type === 'function'); - assertEquals(injected.name, `${SHIM_TOOL_NAME}_2`); + assertEquals(injected.name, `${FUNCTION_TOOL_NAME}_2`); assertEquals(inv.payload.input[1], item); assertEquals(backend.calls, []); }); @@ -3548,7 +3548,7 @@ test('a historical namespace child keeps its identity beside an aliased hosted f type: 'additional_tools', role: 'developer', tools: [{ type: 'namespace', name: 'client', description: 'Client tools', - tools: [{ type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object' } }], + tools: [{ type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object' } }], }], }, ], @@ -3556,11 +3556,11 @@ test('a historical namespace child keeps its identity beside an aliased hosted f }); const history = structuredClone(inv.payload.input); const script = scriptedRun([messageTurn('done')]); - const { result } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + const { result } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'events'); const injected = inv.payload.tools?.[0]; assert(injected?.type === 'function'); - assertEquals(injected.name, `${SHIM_TOOL_NAME}_2`); + assertEquals(injected.name, `${FUNCTION_TOOL_NAME}_2`); assertEquals(inv.payload.input, history); }); @@ -3568,7 +3568,7 @@ test('a historical namespace child keeps its identity beside an aliased hosted f test('upstream returning no actionable events closes the response cleanly', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Turn with only response.created / in_progress / completed. const emptyTurn: ScriptedTurn = [ @@ -3578,7 +3578,7 @@ test('upstream returning no actionable events closes the response cleanly', asyn ]; const script = scriptedRun([emptyTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(script.callCount(), 1); assertEquals(backend.calls.length, 0); @@ -3593,11 +3593,11 @@ test('disabled search provider: dispatched op surfaces explanation snippet (no 5 makeStubDeps({ configured: { type: 'disabled' }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([searchCallTurn(0, 'call_1', 'q'), messageTurn('done', 0)]); - const { result, frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { result, frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'events'); // Two upstream turns still ran — replay-only / mid-conversation @@ -3616,11 +3616,11 @@ test('missing-credential search provider: dispatched op surfaces explanation sni makeStubDeps({ configured: { type: 'missing-credential', provider: 'tavily' }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([searchCallTurn(0, 'call_1', 'q'), messageTurn('done', 0)]); - const { result, frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { result, frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'events'); assertEquals(script.callCount(), 2); @@ -3634,12 +3634,12 @@ test('missing-credential search provider: dispatched op surfaces explanation sni // ── Streaming-specific behavior ─────────────────────────────────────────── -test('shim yields first turn frames BEFORE later turns resolve', async () => { +test('dispatcher yields first turn frames BEFORE later turns resolve', async () => { // Turn 2's run() never resolves until the test releases its gate. Turn // 1's frames must be drainable from the iterator before that resolves; // otherwise the iterator would block on the gate. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let releaseTurn2: (() => void) | undefined; @@ -3674,7 +3674,7 @@ test('shim yields first turn frames BEFORE later turns resolve', async () => { throw new Error(`unexpected run() call ${runCalls}`); }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const iter = result.events[Symbol.asyncIterator](); @@ -3710,11 +3710,11 @@ test('shim yields first turn frames BEFORE later turns resolve', async () => { test('turn 1 pure-text response streams BEFORE upstream terminal (no TTFT regression)', async () => { // A turn-1 pure-text response (no tool call) must not block downstream // until `response.completed` arrives — that would regress TTFT on - // every shim-routed request. The preflight stops the moment + // every hosted-tool request. The preflight stops the moment // `response.created` is captured; everything else (output_text deltas, // terminal) streams live through the same iterator. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Gate the upstream terminal frame so a buffered implementation would // be stuck waiting on it before yielding any byte downstream. @@ -3741,7 +3741,7 @@ test('turn 1 pure-text response streams BEFORE upstream terminal (no TTFT regres return { type: 'events', events: frames, modelIdentity: testTelemetryModelIdentity }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const iter = result.events[Symbol.asyncIterator](); @@ -3764,12 +3764,12 @@ test('turn 1 pure-text response streams BEFORE upstream terminal (no TTFT regres test('mid-stream upstream error yields response.failed and closes the SSE stream', async () => { // After turn 1 emits a web_search call, turn 2's run() returns an - // upstream-error envelope. The shim must yield a single terminal + // upstream-error envelope. The dispatcher must yield a single terminal // response.failed frame and end the iterator — switching the outer // envelope to upstream-error is impossible: the outer envelope shape // is locked once the first frame is yielded. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -3794,7 +3794,7 @@ test('mid-stream upstream error yields response.failed and closes the SSE stream }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -3803,7 +3803,7 @@ test('mid-stream upstream error yields response.failed and closes the SSE stream const failedEv = terminal as Extract; assertEquals(failedEv.response.status, 'failed'); assert(failedEv.response.error !== undefined); - // Pass-through code: the shim quotes upstream's HTTP status + // Pass-through code: the dispatcher quotes upstream's HTTP status // (`upstream_`) when the body has no OpenAI-shaped error // envelope. No normalization to a spec enum value. assertEquals(failedEv.response.error?.code, 'upstream_503'); @@ -3813,13 +3813,13 @@ test('mid-stream upstream error yields response.failed and closes the SSE stream }); test('mid-stream 429 pass-through: code reflects upstream HTTP status (no spec-enum normalization)', async () => { - // The shim no longer normalizes upstream error codes to the + // The dispatcher no longer normalizes upstream error codes to the // OpenAPI `OpenAIResponsesErrorCode` enum. A 429 with no OpenAI-shaped // body falls back to `upstream_429` — the HTTP status is the most // honest signal, and downstream clients pattern-matching on // upstream's actual code see it directly. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -3840,7 +3840,7 @@ test('mid-stream 429 pass-through: code reflects upstream HTTP status (no spec-e }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; @@ -3848,12 +3848,12 @@ test('mid-stream 429 pass-through: code reflects upstream HTTP status (no spec-e }); test('mid-stream 400 with non-OpenAI body falls back to upstream_400 code (no spec-enum normalization)', async () => { - // Without an OpenAI-shaped `error.code` in the body, the shim + // Without an OpenAI-shaped `error.code` in the body, the dispatcher // synthesizes `upstream_` rather than mapping to a spec // enum value. The body excerpt and status live in `error.message` // so clients still see the upstream detail. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -3874,7 +3874,7 @@ test('mid-stream 400 with non-OpenAI body falls back to upstream_400 code (no sp }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; @@ -3891,7 +3891,7 @@ test('mid-stream upstream error with OpenAI-shaped JSON body forwards code/type/ // envelope, pass it through unchanged so SDKs see the same vocabulary // they'd see hitting upstream directly. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -3919,7 +3919,7 @@ test('mid-stream upstream error with OpenAI-shaped JSON body forwards code/type/ }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; @@ -3931,12 +3931,12 @@ test('mid-stream upstream error with OpenAI-shaped JSON body forwards code/type/ test('upstream response.incomplete forwards as response.incomplete with the same incomplete_details.reason', async () => { // Native upstreams emit response.incomplete when the model stopped // before producing a terminal message (max_output_tokens, - // content_filter, ...). The shim used to rewrite it as + // content_filter, ...). The dispatcher used to rewrite it as // response.completed, silently destroying the reason field that // clients branch on. Forward the frame as-is with upstream's reason // preserved. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const incompleteTurn: ScriptedTurn = [ @@ -3948,7 +3948,7 @@ test('upstream response.incomplete forwards as response.incomplete with the same ]; const script = scriptedRun([incompleteTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -3962,13 +3962,13 @@ test('upstream response.incomplete forwards as response.incomplete with the same }); test('upstream response.incomplete WITHOUT incomplete_details forwards as response.incomplete with whatever upstream emitted (no synthetic fold to response.failed)', async () => { - // Pass-through contract for `response.incomplete`: the shim + // Pass-through contract for `response.incomplete`: the dispatcher // forwards upstream's emission verbatim. If upstream sent // `incomplete_details: null`, the downstream wire keeps null; // synthesizing a different terminal kind would lie about what // upstream said. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const incompleteTurn: ScriptedTurn = [ @@ -3980,7 +3980,7 @@ test('upstream response.incomplete WITHOUT incomplete_details forwards as respon ]; const script = scriptedRun([incompleteTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -3993,7 +3993,7 @@ test('upstream response.incomplete WITHOUT incomplete_details forwards as respon test('upstream response.incomplete after a hosted tool call keeps synthesized tool output in the terminal response', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const argsJson = JSON.stringify({ search_query: [{ q: 'q1' }] }); const incompleteTurn: ScriptedTurn = [ @@ -4001,14 +4001,14 @@ test('upstream response.incomplete after a hosted tool call keeps synthesized to mkResponseInProgress(), mkMessageAdded(0), mkMessageDone(0, 'partial answer'), - mkFunctionCallAdded(1, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(1, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(1, argsJson), - mkFunctionCallDone(1, 'call_ws', SHIM_TOOL_NAME, argsJson), + mkFunctionCallDone(1, 'call_ws', FUNCTION_TOOL_NAME, argsJson), mkResponseIncomplete({ reason: 'max_output_tokens' }), ]; const script = scriptedRun([incompleteTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const terminal = events.at(-1) as Extract; @@ -4029,7 +4029,7 @@ const lastFunctionCallOutput = (input: OpenAIResponsesInputItem[]): string => { test('responses target with flag on: function_call_output is plain-text formatted search results', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']), @@ -4039,7 +4039,7 @@ test('responses target with flag on: function_call_output is plain-text formatte messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const text = lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]); assert(text.startsWith('Search results for "q1":')); @@ -4079,11 +4079,11 @@ test('responses target with OpenAI passthrough forwards the complete alpha-searc response_length: 'long', }; const script = scriptedRun([ - fcTurn(0, 'call_1', SHIM_TOOL_NAME, JSON.stringify(commands)), + fcTurn(0, 'call_1', FUNCTION_TOOL_NAME, JSON.stringify(commands)), messageTurn('done', 0), ]); - await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); assertEquals(lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]), 'alpha output'); assertEquals(call.mock.calls[0]?.[0].commands, commands); @@ -4105,10 +4105,10 @@ test('local and cascaded Floway unsupported commands produce the same agent-visi enabledFlags: new Set(['openai-responses-web-search-shim']), }); const script = scriptedRun([ - fcTurn(0, 'call_unsupported', SHIM_TOOL_NAME, JSON.stringify(commands)), + fcTurn(0, 'call_unsupported', FUNCTION_TOOL_NAME, JSON.stringify(commands)), messageTurn('model repeated the tool error', 0), ]); - await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); return lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]); }; @@ -4136,14 +4136,14 @@ test('local and cascaded Floway unsupported commands produce the same agent-visi test('openai-chat-completions target: function_call_output is plain-text formatted search results', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiChatCompletions' }); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const text = lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]); assert(text.startsWith('Search results for "q1":')); @@ -4151,14 +4151,14 @@ test('openai-chat-completions target: function_call_output is plain-text formatt test('messages target: function_call_output is plain-text formatted search results', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'anthropicMessages' }); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const text = lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]); assert(text.startsWith('Search results for "q1":')); @@ -4170,7 +4170,7 @@ test('tool_choice "required" demotes to "auto" after first intercepted turn', as // Without demote: the model is required to call a tool every turn, // never produces a terminal message, loops until the iteration cap. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tool_choice: 'required' } }); // Capture tool_choice on each run() call so we can verify the payload // mutation before draining events. @@ -4184,17 +4184,17 @@ test('tool_choice "required" demotes to "auto" after first intercepted turn', as return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); assertEquals(seenToolChoices[0], 'required'); assertEquals(seenToolChoices[1], 'auto'); }); -test('tool_choice {type:"function", name:} demotes to "auto" after first intercepted turn', async () => { +test('tool_choice {type:"function", name:} demotes to "auto" after first intercepted turn', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ - payload: { tool_choice: { type: 'function', name: SHIM_TOOL_NAME } }, + payload: { tool_choice: { type: 'function', name: FUNCTION_TOOL_NAME } }, }); const seenToolChoices: unknown[] = []; const baseScript = scriptedRun([ @@ -4206,15 +4206,15 @@ test('tool_choice {type:"function", name:} demotes to "auto" aft return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); - assertEquals(seenToolChoices[0], { type: 'function', name: SHIM_TOOL_NAME }); + assertEquals(seenToolChoices[0], { type: 'function', name: FUNCTION_TOOL_NAME }); assertEquals(seenToolChoices[1], 'auto'); }); test('hosted {type:"web_search_preview"} tool_choice gets rewritten and then demoted', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tool_choice: { type: 'web_search_preview' }, @@ -4230,15 +4230,15 @@ test('hosted {type:"web_search_preview"} tool_choice gets rewritten and then dem return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); - assertEquals(seenToolChoices[0], { type: 'function', name: SHIM_TOOL_NAME }); + assertEquals(seenToolChoices[0], { type: 'function', name: FUNCTION_TOOL_NAME }); assertEquals(seenToolChoices[1], 'auto'); }); test('tool_choice "auto" stays "auto" — no demotion when never forced', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tool_choice: 'auto' } }); const seenToolChoices: unknown[] = []; const baseScript = scriptedRun([ @@ -4250,15 +4250,15 @@ test('tool_choice "auto" stays "auto" — no demotion when never forced', async return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); assertEquals(seenToolChoices[0], 'auto'); assertEquals(seenToolChoices[1], 'auto'); }); -test.each([false, true])('forced namespaced client tool choice is not mistaken for the hosted shim function (declared %s)', async declared => { +test.each([false, true])('forced namespaced client tool choice is not mistaken for the hosted dispatcher function (declared %s)', async declared => { const { backend } = makeStubDeps(); - const choice = { type: 'function' as const, namespace: 'client', name: SHIM_TOOL_NAME }; + const choice = { type: 'function' as const, namespace: 'client', name: FUNCTION_TOOL_NAME }; const inv = makeInvocation({ payload: { tool_choice: choice, @@ -4266,25 +4266,25 @@ test.each([false, true])('forced namespaced client tool choice is not mistaken f { type: 'web_search' }, ...(declared ? [{ type: 'namespace', name: 'client', description: 'Client tools', - tools: [{ type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object' } }], + tools: [{ type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object' } }], } satisfies OpenAIResponsesTool] : []), ], }, }); const seenToolChoices: unknown[] = []; const script = scriptedRun([ - fcTurn(0, 'call_search', `${SHIM_TOOL_NAME}_2`, JSON.stringify({ search_query: [{ q: 'q1' }] })), + fcTurn(0, 'call_search', `${FUNCTION_TOOL_NAME}_2`, JSON.stringify({ search_query: [{ q: 'q1' }] })), messageTurn('done'), ]); const run = async () => { const injected = inv.payload.tools?.[0]; assert(injected?.type === 'function'); - assertEquals(injected.name, `${SHIM_TOOL_NAME}_2`); + assertEquals(injected.name, `${FUNCTION_TOOL_NAME}_2`); seenToolChoices.push(inv.payload.tool_choice); return await script.run(); }; - await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), run); + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), run); assertEquals(seenToolChoices, [choice, choice]); assertEquals(backend.calls.length, 1); @@ -4297,7 +4297,7 @@ test('cap-exceeded does NOT set tool_choice="none" — the cap snippet alone nud // path relies solely on the exhausted-budget snippet to nudge the model // to switch tools or settle on a terminal message. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tool_choice: 'auto' } }); // 30 search turns, then turn 31 (cap-exceeded), then a final message. const searchTurns: ScriptedTurn[] = []; @@ -4312,7 +4312,7 @@ test('cap-exceeded does NOT set tool_choice="none" — the cap snippet alone nud return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); // Every turn observed `'auto'` — no demotion to 'none' after the cap. assertEquals(seenToolChoices[30], 'auto'); @@ -4321,7 +4321,7 @@ test('cap-exceeded does NOT set tool_choice="none" — the cap snippet alone nud test('max_tool_calls is forwarded to upstream turns but does not locally bypass hosted tools', async () => { const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { max_tool_calls: 1 } }); const seenMaxToolCalls: unknown[] = []; const baseScript = scriptedRun([ @@ -4334,7 +4334,7 @@ test('max_tool_calls is forwarded to upstream turns but does not locally bypass return await baseScript.run(); }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); await collectFrames(result.events); assertEquals(seenMaxToolCalls, [1, 0, 0]); @@ -4343,11 +4343,11 @@ test('max_tool_calls is forwarded to upstream turns but does not locally bypass test('max_tool_calls invalid values pass through to upstream and do not engage local accounting', async () => { makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { max_tool_calls: 'one' as unknown as number } }); const script = scriptedRun([messageTurn('hello', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); assertEquals(events[events.length - 1].type, 'response.completed'); @@ -4361,7 +4361,7 @@ test('final-turn text deltas stream as they arrive, not buffered until response. // first delta is drainable while the gate is still closed proves true // byte-by-byte streaming. makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let releaseRestOfTurn2: (() => void) | undefined; @@ -4409,7 +4409,7 @@ test('final-turn text deltas stream as they arrive, not buffered until response. throw new Error(`unexpected run() call ${runCalls}`); }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const iter = result.events[Symbol.asyncIterator](); @@ -4436,7 +4436,7 @@ test('mixed turn (reasoning + message_partial + function_call) preserves the mes // [reasoning, message, function_call]. The message forwards live and // the synthesized web_search_call follows it. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const mixedTurn: ScriptedTurn = [ @@ -4453,9 +4453,9 @@ test('mixed turn (reasoning + message_partial + function_call) preserves the mes delta: 'thinking out loud', }), mkMessageDone(1, 'thinking out loud'), - mkFunctionCallAdded(2, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(2, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(2, JSON.stringify({ search_query: [{ q: 'q1' }] })), - mkFunctionCallDone(2, 'call_ws', SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'q1' }] })), + mkFunctionCallDone(2, 'call_ws', FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'q1' }] })), mkResponseCompleted(), ]; const script = scriptedRun([mixedTurn, messageTurn('final answer', 0)]); @@ -4465,7 +4465,7 @@ test('mixed turn (reasoning + message_partial + function_call) preserves the mes return await script.run(); }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -4493,7 +4493,7 @@ test('two consecutive tool-call turns each with a thinking-out-loud message pres // before dispatching its tool call. Both intermediate messages forward // live and survive in the downstream output. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const thinkingTurn = ( @@ -4509,9 +4509,9 @@ test('two consecutive tool-call turns each with a thinking-out-loud message pres mkResponseInProgress(), mkMessageAdded(msgUpstreamIdx), mkMessageDone(msgUpstreamIdx, msgText), - mkFunctionCallAdded(fcUpstreamIdx, callId, SHIM_TOOL_NAME), + mkFunctionCallAdded(fcUpstreamIdx, callId, FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(fcUpstreamIdx, args), - mkFunctionCallDone(fcUpstreamIdx, callId, SHIM_TOOL_NAME, args), + mkFunctionCallDone(fcUpstreamIdx, callId, FUNCTION_TOOL_NAME, args), mkResponseCompleted(), ]; }; @@ -4522,7 +4522,7 @@ test('two consecutive tool-call turns each with a thinking-out-loud message pres messageTurn('final answer', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -4550,7 +4550,7 @@ test('two consecutive tool-call turns each with a thinking-out-loud message pres test('lifecycle start frames yield BEFORE backend resolves, giving searching real wall-clock duration', async () => { // Gate provider.search() so the test can observe ordering: lifecycle // start frames must be drainable WHILE the backend is pending. If the - // shim awaits the backend before yielding start frames, next() after + // dispatcher awaits the backend before yielding start frames, next() after // response.created will hang on the gate. let releaseSearch: (() => void) | null = null; const searchGate = new Promise(resolve => { releaseSearch = resolve; }); @@ -4569,14 +4569,14 @@ test('lifecycle start frames yield BEFORE backend resolves, giving searching rea }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'gated-q'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const iter = result.events[Symbol.asyncIterator](); @@ -4595,7 +4595,7 @@ test('lifecycle start frames yield BEFORE backend resolves, giving searching rea if (f.event.type === 'response.web_search_call.completed') sawCompleted = true; } assertEquals(sawSearching, true); - // Critical: searching arrived but completed did NOT — the shim hasn't + // Critical: searching arrived but completed did NOT — the dispatcher hasn't // awaited the backend yet, but the wire already shows searching. assertEquals(sawCompleted, false); // The backend call IS in flight (awaiting the gate); the call was placed @@ -4614,13 +4614,13 @@ test('lifecycle start frames yield BEFORE backend resolves, giving searching rea assert(types.includes('response.output_item.done')); }); -test('terminal response.completed.output is in output_index order, not completion order (shim backend resolves AFTER a later live item)', async () => { - // The shim call reserves downstream index 0 at output_item.added time; +test('terminal response.completed.output is in output_index order, not completion order (dispatcher backend resolves AFTER a later live item)', async () => { + // The dispatcher call reserves downstream index 0 at output_item.added time; // the mixed-tool client function_call gets downstream index 1. - // We gate the backend so the shim's output_item.done fires AFTER + // We gate the backend so the dispatcher's output_item.done fires AFTER // the client function_call has already finalized into accumulatedOutput. // The sparse-index materialization at terminal time must still place - // the shim call at output[0] and the client tool at output[1]. + // the dispatcher call at output[0] and the client tool at output[1]. let releaseSearch: (() => void) | null = null; const searchGate = new Promise(resolve => { releaseSearch = resolve; }); makeStubDeps({ @@ -4638,67 +4638,67 @@ test('terminal response.completed.output is in output_index order, not completio }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const wsArgs = JSON.stringify({ search_query: [{ q: 'q' }] }); - // Shim call at upstream index 0, client tool at upstream index 1. The + // Dispatcher call at upstream index 0, client tool at upstream index 1. The // upstream's response.completed arrives BEFORE the gated backend - // resolves; the shim will release end frames lazily as the consumer + // resolves; the dispatcher will release end frames lazily as the consumer // pulls (so we must release the gate to finish the stream). const mixedTurn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, wsArgs), - mkFunctionCallDone(0, 'call_ws', SHIM_TOOL_NAME, wsArgs), + mkFunctionCallDone(0, 'call_ws', FUNCTION_TOOL_NAME, wsArgs), mkFunctionCallAdded(1, 'call_other', 'lookup'), mkFunctionCallArgsDone(1, '{"q":"x"}', 'fc_1'), mkFunctionCallDone(1, 'call_other', 'lookup', '{"q":"x"}'), mkResponseCompleted(), ]; const script = scriptedRun([mixedTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); - // Release the gate so the shim call end frames + terminal can resolve. + // Release the gate so the dispatcher call end frames + terminal can resolve. releaseSearch!(); const events = eventPayloads(await collectFrames(result.events)); const terminal = events[events.length - 1]; assertEquals(terminal.type, 'response.completed'); const output = (terminal as { response: { output: Array<{ type: string }> } }).response.output; - // Two items: shim web_search_call at slot 0, client function_call at slot 1. + // Two items: dispatcher web_search_call at slot 0, client function_call at slot 1. // Reserved-slot ordering wins even though the client tool finalized - // first (no await on the .done path) and the shim call finalized last. + // first (no await on the .done path) and the dispatcher call finalized last. assertEquals(output.map(o => o.type), ['web_search_call', 'function_call']); }); // ── End-to-end protocol-violation paths ────────────────────────────── -test('shim call without output_item.done synthesizes response.failed (no backend dispatch)', async () => { - // Protocol violation: upstream emits the shim's added + - // arguments deltas but no `.done`. The unmatched-shim-call detector +test('dispatcher call without output_item.done synthesizes response.failed (no backend dispatch)', async () => { + // Protocol violation: upstream emits the dispatcher's added + + // arguments deltas but no `.done`. The unmatched-hosted-call detector // in consume-turn must promote the turn to `response.failed`; no - // backend search / fetchPage should fire (the shim call never + // backend search / fetchPage should fire (the dispatcher call never // dispatched). const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const argsJson = JSON.stringify({ search_query: [{ q: 'truncated' }] }); const script = scriptedRun([[ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_truncated', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_truncated', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, argsJson), // No `mkFunctionCallDone` — upstream protocol violation. mkResponseCompleted(), ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const terminal = events[events.length - 1] as Extract; assertEquals(terminal.type, 'response.failed'); assertEquals(terminal.response.error?.code, 'server_error'); - assert(terminal.response.error?.message.includes('without closing shim call items')); + assert(terminal.response.error?.message.includes('without closing dispatcher call items')); // Backend never invoked — dispatch only fires from output_item.done. assertEquals(backend.calls.length, 0); }); @@ -4710,16 +4710,16 @@ test('wrong-typed supported sub-property (search_query as object) synthesizes a // a "wrong-type sub-property" snippet) instead of crashing the // dispatcher. Backend search should NOT fire. const { backend } = makeStubDeps(); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Pass `search_query` as a single object — wrong type. const wrongArgs = JSON.stringify({ search_query: { q: 'x' } }); const script = scriptedRun([ - fcTurn(0, 'call_wrong', SHIM_TOOL_NAME, wrongArgs), + fcTurn(0, 'call_wrong', FUNCTION_TOOL_NAME, wrongArgs), messageTurn('giving up', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const frames = await collectFrames(result.events); const doneEvents = outputItemDoneEvents(frames); @@ -4737,14 +4737,14 @@ test('wrong-typed supported sub-property (search_query as object) synthesizes a }); test('downstream AbortSignal threads through to provider search / fetchPage and propagates aborts', async () => { - // Cancelled requests must stop generating upstream load. The shim + // Cancelled requests must stop generating upstream load. The dispatcher // threads the request's `downstreamAbortSignal` into every backend // provider call so providers can observe and abort. let observedSignal: AbortSignal | undefined; const controller = new AbortController(); const { backend } = makeStubDeps({ providerOverrides: { - // Capture the signal the shim hands us; resolve only when the + // Capture the signal the dispatcher hands us; resolve only when the // signal aborts so we can assert downstream propagation. async search(request) { observedSignal = request.signal; @@ -4760,7 +4760,7 @@ test('downstream AbortSignal threads through to provider search / fetchPage and }, }, }); - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const gatewayCtx = mockChatGatewayCtx({ apiKeyId: 'k1', @@ -4771,7 +4771,7 @@ test('downstream AbortSignal threads through to provider search / fetchPage and searchCallTurn(0, 'call_1', 'will-be-aborted'), ]); - const result = await shim(inv, gatewayCtx, script.run); + const result = await dispatcher(inv, gatewayCtx, script.run); assert(result.type === 'events'); // Drain the events stream in the background. The backend search // will hang until we abort, so the drain promise won't resolve @@ -4823,7 +4823,7 @@ interface DispatchRecord { intercepted: InterceptedFunctionCall; } -// Records every shim call without producing IRs or start frames. +// Records every dispatcher call without producing IRs or start frames. // Tests that care about dispatcher behavior pass a custom dispatcher. const recordingDispatcher = (records: DispatchRecord[]) => ({ intercepted }: { intercepted: InterceptedFunctionCall }) => { records.push({ intercepted }); @@ -4877,15 +4877,15 @@ const consumeTurn = async ( ): Promise => { const records: DispatchRecord[] = []; return await drain( - consumeTurnStreaming(frames, state, isFirstTurn, new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []), + consumeTurnStreaming(frames, state, isFirstTurn, new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []), records, ); }; -test('consumeTurn forwards a namespaced client call sharing the hosted shim name', async () => { +test('consumeTurn forwards a namespaced client call sharing the hosted dispatcher name', async () => { const call = { type: 'function_call' as const, - id: 'fc_client', call_id: 'call_client', namespace: 'client', name: SHIM_TOOL_NAME, + id: 'fc_client', call_id: 'call_client', namespace: 'client', name: FUNCTION_TOOL_NAME, arguments: '{"query":"client data"}', status: 'completed', }; const result = await consumeTurn(framesOf( @@ -4908,7 +4908,7 @@ test('consumeTurn forwards a namespaced client call sharing the hosted shim name test('consumeTurn uses the completed call namespace before executing a hosted tool', async () => { const completed = { type: 'function_call' as const, - id: 'fc_client', call_id: 'call_client', namespace: 'client', name: SHIM_TOOL_NAME, + id: 'fc_client', call_id: 'call_client', namespace: 'client', name: FUNCTION_TOOL_NAME, arguments: '{"client":true}', status: 'completed', }; const result = await consumeTurn(framesOf( @@ -4939,20 +4939,20 @@ test('consumeTurn dispatches a hosted call whose added item carried a namespace' type: 'response.output_item.added', output_index: 0, item: { type: 'function_call', id: 'fc_search', call_id: 'call_search', namespace: 'client', - name: SHIM_TOOL_NAME, arguments: '', status: 'in_progress', + name: FUNCTION_TOOL_NAME, arguments: '', status: 'in_progress', }, }), eventFrame({ type: 'response.output_item.done', output_index: 0, item: { type: 'function_call', id: 'fc_search', call_id: 'call_search', - name: SHIM_TOOL_NAME, arguments: '{"search_query":[]}', status: 'completed', + name: FUNCTION_TOOL_NAME, arguments: '{"search_query":[]}', status: 'completed', }, }), mkResponseCompleted(), ), createMergeState(), true); - assertEquals(result.records.map(record => record.intercepted.name), [SHIM_TOOL_NAME]); + assertEquals(result.records.map(record => record.intercepted.name), [FUNCTION_TOOL_NAME]); assertEquals(result.summary.sawClientToolCall, false); assertEquals(outputItemDoneEvents(result.downstreamFrames), []); }); @@ -4990,10 +4990,10 @@ test('consumeTurn first turn synthesizes response.created with the once-per-requ const created = result.downstreamFrames[0]; assert(created.type === 'event'); const createdEv = created.event as Extract; - // Downstream id is the shim-synthesized value (stable cross-turn); + // Downstream id is the gateway-synthesized value (stable cross-turn); // upstream's id is captured nowhere and never exposed downstream. assertEquals(createdEv.response.id, state.synthesizedResponseId); - assert(state.synthesizedResponseId.startsWith('resp_shim_')); + assert(state.synthesizedResponseId.startsWith('resp_hosted_')); }); test('consumeTurn synthesizes response.created with the upstream-reported model (no client fallback)', async () => { @@ -5034,7 +5034,7 @@ test('consumeTurn throws when upstream response.created has no model field (no c ), state, true, - new Map([[SHIM_TOOL_NAME, recordingDispatcher([])]]), + new Map([[FUNCTION_TOOL_NAME, recordingDispatcher([])]]), loopState(), [], ); @@ -5091,8 +5091,8 @@ test('consumeTurn re-captures upstream-reported model when later turns change it assertEquals(state.lastSeenModel, 'gpt-5.6-2025-12-01'); }); -test('consumeTurn does NOT capture upstream response.id (downstream uses the shim-synthesized id only)', async () => { - // Upstream's id rotates per turn and the shim never exposes it +test('consumeTurn does NOT capture upstream response.id (downstream uses the gateway-synthesized id only)', async () => { + // Upstream's id rotates per turn and the dispatcher never exposes it // downstream — `synthesizedResponseId` is the single cross-turn // identity the client correlates against. Verify upstream's id // doesn't slip into any MergeState field. @@ -5150,17 +5150,17 @@ test('consumeTurn second turn swallows upstream response.created and in_progress assertEquals(eventTypesOf(result.downstreamFrames), []); }); -test('consumeTurn intercepts the shim tool and does NOT forward its 4 events', async () => { +test('consumeTurn intercepts the dispatcher tool and does NOT forward its 4 events', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDelta(0, '{"search_q'), mkFunctionCallArgsDelta(0, 'uery":[{"q":"hello"}]}'), mkFunctionCallArgsDone(0, '{"search_query":[{"q":"hello"}]}'), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"hello"}]}'), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"hello"}]}'), mkResponseCompleted(), ), state, @@ -5170,7 +5170,7 @@ test('consumeTurn intercepts the shim tool and does NOT forward its 4 events', a assertEquals(result.records.length, 1); assertEquals(result.records[0].intercepted, { callId: 'cc_1', - name: SHIM_TOOL_NAME, + name: FUNCTION_TOOL_NAME, arguments: { search_query: [{ q: 'hello' }] }, }); assertEquals(result.summary.dispatched.length, 1); @@ -5186,15 +5186,15 @@ test('consumeTurn intercepts the shim tool and does NOT forward its 4 events', a assertFalse(result.summary.sawClientToolCall); }); -test('consumeTurn intercepts two shim calls within one turn', async () => { +test('consumeTurn intercepts two dispatcher calls within one turn', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_o', SHIM_TOOL_NAME), - mkFunctionCallDone(0, 'cc_o', SHIM_TOOL_NAME, '{"open":[{"ref_id":"https://x"}]}'), - mkFunctionCallAdded(1, 'cc_f', SHIM_TOOL_NAME), - mkFunctionCallDone(1, 'cc_f', SHIM_TOOL_NAME, '{"find":[{"ref_id":"https://x","pattern":"p"}]}'), + mkFunctionCallAdded(0, 'cc_o', FUNCTION_TOOL_NAME), + mkFunctionCallDone(0, 'cc_o', FUNCTION_TOOL_NAME, '{"open":[{"ref_id":"https://x"}]}'), + mkFunctionCallAdded(1, 'cc_f', FUNCTION_TOOL_NAME), + mkFunctionCallDone(1, 'cc_f', FUNCTION_TOOL_NAME, '{"find":[{"ref_id":"https://x","pattern":"p"}]}'), mkResponseCompleted(), ), state, @@ -5205,8 +5205,8 @@ test('consumeTurn intercepts two shim calls within one turn', async () => { assertEquals(result.records[1].intercepted.arguments, { find: [{ ref_id: 'https://x', pattern: 'p' }] }); }); -test('consumeTurn synthesizes response.failed when upstream terminates without closing a shim call', async () => { - // A shim call reservation that never receives `output_item.done` is +test('consumeTurn synthesizes response.failed when upstream terminates without closing a dispatcher call', async () => { + // A dispatcher call reservation that never receives `output_item.done` is // an upstream protocol violation: the model intended a tool call, // the gateway accepted the reservation, but the close frame never // arrived. Without explicit detection here the reservation is @@ -5220,7 +5220,7 @@ test('consumeTurn synthesizes response.failed when upstream terminates without c consumeTurnStreaming( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDelta(0, '{"x":'), mkFunctionCallArgsDelta(0, '1}'), // No function_call.done — dispatcher never fires. @@ -5228,7 +5228,7 @@ test('consumeTurn synthesizes response.failed when upstream terminates without c ), state, true, - new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), + new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), [], ), @@ -5237,7 +5237,7 @@ test('consumeTurn synthesizes response.failed when upstream terminates without c assertEquals(result.summary.dispatched.length, 0); assertEquals(result.summary.terminalStatus.kind, 'failed'); const ts = result.summary.terminalStatus as Extract; - assert(ts.response.error!.message.includes('without closing shim call items')); + assert(ts.response.error!.message.includes('without closing dispatcher call items')); assert(ts.response.error!.message.includes('response.completed')); }); @@ -5246,11 +5246,11 @@ test('consumeTurn dispatches at function_call.done with .done args canonical ove const result = await consumeTurn( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDelta(0, '{"stale":'), mkFunctionCallArgsDelta(0, '1}'), mkFunctionCallArgsDone(0, '{"search_query":[{"q":"x"}]}'), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"x"}]}'), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"x"}]}'), mkResponseCompleted(), ), state, @@ -5259,7 +5259,7 @@ test('consumeTurn dispatches at function_call.done with .done args canonical ove assertEquals(result.records[0].intercepted.arguments, { search_query: [{ q: 'x' }] }); }); -test('consumeTurn live-forwards non-shim function_calls and sets sawClientToolCall', async () => { +test('consumeTurn live-forwards client-owned function_calls and sets sawClientToolCall', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( @@ -5381,14 +5381,14 @@ test('consumeTurn single iteration ending in message: forwards full message life ]); }); -test('consumeTurn one shim call then message in same turn: FORWARDS the message live (shim call is consumed)', async () => { +test('consumeTurn one dispatcher call then message in same turn: FORWARDS the message live (dispatcher call is consumed)', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"hi"}]}'), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"hi"}]}'), mkMessageAdded(1), mkMessageDone(1, 'intermediate text'), mkResponseCompleted(), @@ -5399,7 +5399,7 @@ test('consumeTurn one shim call then message in same turn: FORWARDS the message assertEquals(result.records.length, 1); assertEquals(state.accumulatedOutput.size, 1); - // Recording dispatcher doesn't emit lifecycle frames so the shim's + // Recording dispatcher doesn't emit lifecycle frames so the dispatcher's // reserved slot stays empty in accumulatedOutput, but outputIndex was // still bumped. The message therefore lands at index 1. assertEquals(state.accumulatedOutput.get(1)?.type, 'message'); @@ -5531,14 +5531,14 @@ test('consumeTurn swallows future indexed events attached to an intercepted host const result = await consumeTurn( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), eventFrame({ type: 'response.future_function_call_arguments.delta', item_id: 'fc_hidden', output_index: 0, delta: 'hidden', } as unknown as OpenAIResponsesStreamEvent), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"q"}]}'), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"q"}]}'), mkResponseCompleted(), ), state, @@ -5602,8 +5602,8 @@ test('consumeTurn forwards message text events live even when mixed with an inte const result = await consumeTurn( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"q"}]}'), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"q"}]}'), mkMessageAdded(1), eventFrame({ type: 'response.output_text.delta', @@ -5800,7 +5800,7 @@ test('consumeTurn surfaces bare `error` event as terminalStatus.failed with a sy assertEquals(ts.response.status, 'failed'); assertEquals(ts.response.error?.message, 'upstream blew up'); assertEquals(ts.response.error?.code, 'server_error'); - // Synthesized envelope's id is the shim-synthesized response id + // Synthesized envelope's id is the gateway-synthesized response id // — upstream's id is not exposed downstream. assertEquals(ts.response.id, state.synthesizedResponseId); }); @@ -5866,7 +5866,7 @@ test('consumeTurn surfaces bare `error` event arriving BEFORE response.created a // identity is captured (truncated TLS, transport drop, intermediate // proxy injection), we cannot synthesize a wire-valid // `OpenAIResponsesResult` (id and model are required, not nullable). Use - // a distinct terminal status `bare-error-pre-shell` so the shim's + // a distinct terminal status `bare-error-pre-shell` so the dispatcher's // outer loop can short-circuit to a non-events `upstream-error` // result instead of fabricating empty-string identity fields. const state = createMergeState(); @@ -5911,7 +5911,7 @@ test('consumeTurnStreaming yields forwarded frames before upstream completes', a countedFrames, state, true, - new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), + new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), [], ); @@ -5925,7 +5925,7 @@ test('consumeTurnStreaming yields forwarded frames before upstream completes', a while (!(await iter.next()).done) { /* drain */ } }); -test('dispatcher start frames yield IN-LINE at function_call.done (shim call slot precedes later items)', async () => { +test('dispatcher start frames yield IN-LINE at function_call.done (dispatcher call slot precedes later items)', async () => { const state = createMergeState(); let dispatchOrder = 0; const records: DispatchRecord[] = []; @@ -5949,15 +5949,15 @@ test('dispatcher start frames yield IN-LINE at function_call.done (shim call slo consumeTurnStreaming( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"hi"}]}'), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"hi"}]}'), mkMessageAdded(1), mkMessageDone(1, 'after'), mkResponseCompleted(), ), state, true, - new Map([[SHIM_TOOL_NAME, dispatcher]]), + new Map([[FUNCTION_TOOL_NAME, dispatcher]]), loopState(), [], ), @@ -5975,9 +5975,9 @@ test('dispatcher start frames yield IN-LINE at function_call.done (shim call slo assert(syntheticIdx < messageAddedIdx, `expected dispatcher start frame BEFORE later live items (synth=${syntheticIdx}, msgAdded=${messageAddedIdx})`); }); -test('shim call output_index is reserved at output_item.added so interleaved items get later indices', async () => { - // Reserving at `.added` (rather than `.done`) keeps a non-shim call - // item arriving between added and done from stealing the shim's +test('dispatcher call output_index is reserved at output_item.added so interleaved items get later indices', async () => { + // Reserving at `.added` (rather than `.done`) keeps a client-owned call + // item arriving between added and done from stealing the dispatcher's // would-be downstream index. const state = createMergeState(); const dispatcher = () => [] as HostedToolResultSlot[]; @@ -5993,14 +5993,14 @@ test('shim call output_index is reserved at output_item.added so interleaved ite consumeTurnStreaming( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), interleaved, - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"x"}]}'), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"x"}]}'), mkResponseCompleted(), ), state, true, - new Map([[SHIM_TOOL_NAME, dispatcher]]), + new Map([[FUNCTION_TOOL_NAME, dispatcher]]), loopState(), [], ), @@ -6078,7 +6078,7 @@ test('consumeTurn synthesizes terminalStatus.failed when upstream stream ends wi test('createMergeState starts with empty sparse usage accumulator and a synthesized response id', () => { const s = createMergeState(); - assert(s.synthesizedResponseId.startsWith('resp_shim_')); + assert(s.synthesizedResponseId.startsWith('resp_hosted_')); assertEquals(s.lastSeenModel, null); assertEquals(s.sequenceNumber, 0); assertEquals(s.outputIndex, 0); @@ -6217,7 +6217,7 @@ test('HostedToolResultSlot run() yields nothing and returns the terminal', async // ── Echo restore on `response.tools` / `response.tool_choice` ───────────── // -// The shim rewrites the request's hosted `web_search` to a function tool +// The dispatcher rewrites the request's hosted `web_search` to a function tool // before upstream; on echo (per OpenAI spec, `Response` composes // `ResponseProperties`, so `tools` and `tool_choice` are echoed) the // synthesized envelope must restore the canonical hosted form so the @@ -6262,15 +6262,15 @@ const findResponseCompleted = ( test('echo restore swaps the injected function tool back to the canonical hosted web_search', async () => { makeStubDeps(); - // Upstream's echoed `tools` includes the function tool the shim + // Upstream's echoed `tools` includes the function tool the dispatcher // injected plus an ordinary client function tool — restore should // touch only the injected one. (Spec defaults like // `additionalProperties:false` that Copilot injects on function - // tools must pass through verbatim on the non-shim tool.) + // tools must pass through verbatim on the client-owned tool.) const upstreamEchoedTools: OpenAIResponsesTool[] = [ { type: 'function', - name: SHIM_TOOL_NAME, + name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false, description: 'gateway-internal', @@ -6283,7 +6283,7 @@ test('echo restore swaps the injected function tool back to the canonical hosted description: null as unknown as string, }, ]; - const shim = runHostedWebSearch; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -6300,7 +6300,7 @@ test('echo restore swaps the injected function tool back to the canonical hosted mkResponseCompletedWithTools(upstreamEchoedTools), ]]); - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const completed = findResponseCompleted(frames); assert(Array.isArray(completed.response.tools)); @@ -6318,7 +6318,7 @@ test('echo restore preserves client-supplied filters / user_location on the cano const userLoc = { city: 'Tokyo', country: 'JP' }; const filters = { allowed_domains: ['weather.gov'] }; const upstreamEchoedTools: OpenAIResponsesTool[] = [ - { type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, + { type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, ]; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters, user_location: userLoc }] } }); const script = scriptedRun([[ @@ -6329,7 +6329,7 @@ test('echo restore preserves client-supplied filters / user_location on the cano mkResponseCompletedWithTools(upstreamEchoedTools), ]]); - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); const completed = findResponseCompleted(frames); assertEquals(completed.response.tools![0].type, 'web_search'); @@ -6352,7 +6352,7 @@ test('duplicate hosted web_search declarations collapse to the last complete dec // Upstream sees exactly one function tool because the framework // dedupes before run(). const upstreamEchoedTools: OpenAIResponsesTool[] = [ - { type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, + { type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, ]; const script = scriptedRun([[ mkResponseCreatedWithTools(upstreamEchoedTools), @@ -6362,7 +6362,7 @@ test('duplicate hosted web_search declarations collapse to the last complete dec mkResponseCompletedWithTools(upstreamEchoedTools), ]]); - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); // Request side: only one function tool was sent upstream. assertEquals(inv.payload.tools?.length, 1); @@ -6379,7 +6379,7 @@ test('duplicate hosted web_search declarations collapse to the last complete dec test('echo restore swaps the function-typed tool_choice back to a hosted tool_choice', async () => { makeStubDeps(); const upstreamEchoedTools: OpenAIResponsesTool[] = [ - { type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, + { type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, ]; const inv = makeInvocation({ payload: { @@ -6388,17 +6388,17 @@ test('echo restore swaps the function-typed tool_choice back to a hosted tool_ch }, }); const script = scriptedRun([[ - mkResponseCreatedWithTools(upstreamEchoedTools, { type: 'function', name: SHIM_TOOL_NAME }), + mkResponseCreatedWithTools(upstreamEchoedTools, { type: 'function', name: FUNCTION_TOOL_NAME }), mkResponseInProgress(), mkMessageAdded(0), mkMessageDone(0, 'done'), - mkResponseCompletedWithTools(upstreamEchoedTools, { type: 'function', name: SHIM_TOOL_NAME }), + mkResponseCompletedWithTools(upstreamEchoedTools, { type: 'function', name: FUNCTION_TOOL_NAME }), ]]); - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); // Request side: tool_choice was rewritten to the function form. - assertEquals(inv.payload.tool_choice, { type: 'function', name: SHIM_TOOL_NAME }); + assertEquals(inv.payload.tool_choice, { type: 'function', name: FUNCTION_TOOL_NAME }); const completed = findResponseCompleted(frames); assertEquals(completed.response.tool_choice, { type: 'web_search' }); }); @@ -6406,9 +6406,9 @@ test('echo restore swaps the function-typed tool_choice back to a hosted tool_ch test('echo restore leaves a non-injected function-typed tool_choice untouched', async () => { makeStubDeps(); // Upstream echoes a tool_choice naming a CLIENT-supplied function - // tool, not the shim's. Restore must not rewrite that. + // tool, not the dispatcher's. Restore must not rewrite that. const upstreamEchoedTools: OpenAIResponsesTool[] = [ - { type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, + { type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, { type: 'function', name: 'get_weather', parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, ]; const inv = makeInvocation({ @@ -6428,7 +6428,7 @@ test('echo restore leaves a non-injected function-typed tool_choice untouched', mkResponseCompletedWithTools(upstreamEchoedTools, { type: 'function', name: 'get_weather' }), ]]); - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); const completed = findResponseCompleted(frames); assertEquals(completed.response.tool_choice, { type: 'function', name: 'get_weather' }); @@ -6441,7 +6441,7 @@ test('upstream that does not echo `tools` produces a synthesized envelope withou // upstream that simply omits the echo. const script = scriptedRun([messageTurn('done')]); - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); const completed = findResponseCompleted(frames); assertEquals(completed.response.tools, undefined); @@ -6450,7 +6450,7 @@ test('upstream that does not echo `tools` produces a synthesized envelope withou // ── Billable usage ──────────────────────────────────────────────────────── -test('the shim carries the upstream turn cost onto the reading settlement writes from', async () => { +test('the dispatcher carries the upstream turn cost onto the reading settlement writes from', async () => { // The stage publishes its own reading, and that is what settlement writes the row from. // Failing to carry the cost onto it loses billing for the whole response silently, because // the client-facing usage is no longer consulted. @@ -6478,7 +6478,7 @@ test('the shim carries the upstream turn cost onto the reading settlement writes }); for (const carrier of ['tools', 'additional_tools', 'tool_search_output'] as const) { - test(`the shim drains ${carrier} continuation streams before awaiting billable metadata`, async () => { + test(`the dispatcher drains ${carrier} continuation streams before awaiting billable metadata`, async () => { const { backend } = makeStubDeps(); const tools: OpenAIResponsesTool[] = [{ type: 'web_search' }]; const inv = makeInvocation({ @@ -6515,7 +6515,7 @@ for (const carrier of ['tools', 'additional_tools', 'tool_search_output'] as con } for (const failedTurn of [1, 2]) { - test(`the shim retains observed billing when turn ${failedTurn} throws after its usage`, async () => { + test(`the dispatcher retains observed billing when turn ${failedTurn} throws after its usage`, async () => { makeStubDeps(); let runCalls = 0; const streamError = new Error('stream interrupted after usage'); @@ -6586,7 +6586,7 @@ test('a search runs with its own prologue and its own record', async () => { const inv = makeInvocation({ payload: { tools: [{ type: 'web_search' }] } }); const script = scriptedRun([searchCallTurn(0, 'call_1', 'floway'), messageTurn('done', 0)]); - await runShimAndDrain(runHostedWebSearch, inv, mockChatGatewayCtx({ apiKeyId: 'k1', wantsStream: true, dump: turn }), script.run); + await runHostedToolsAndDrain(runHostedWebSearch, inv, mockChatGatewayCtx({ apiKeyId: 'k1', wantsStream: true, dump: turn }), script.run); await flushBackground(); // One record, and it is the search's rather than the turn's: the turn's own is finalized by @@ -6618,16 +6618,16 @@ test('helper allocation reserves names across callable scopes, history and searc }); const undeclaredSelectorChoices: { name: string; choice: OpenAIResponsesToolChoice }[] = [ - { name: 'forced', choice: { type: 'function', name: `${SHIM_TOOL_NAME}_2` } }, - { name: 'allowed auto', choice: { type: 'allowed_tools', mode: 'auto', tools: [{ type: 'function', name: `${SHIM_TOOL_NAME}_2` }] } }, - { name: 'allowed required', choice: { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: `${SHIM_TOOL_NAME}_2` }] } }, + { name: 'forced', choice: { type: 'function', name: `${FUNCTION_TOOL_NAME}_2` } }, + { name: 'allowed auto', choice: { type: 'allowed_tools', mode: 'auto', tools: [{ type: 'function', name: `${FUNCTION_TOOL_NAME}_2` }] } }, + { name: 'allowed required', choice: { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: `${FUNCTION_TOOL_NAME}_2` }] } }, ]; test.each(undeclaredSelectorChoices)('helper allocation does not capture an undeclared $name selector', async ({ choice }) => { const { backend } = makeStubDeps(); const tools: OpenAIResponsesTool[] = [ { type: 'web_search' }, - { type: 'namespace', name: 'aux', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }, + { type: 'namespace', name: 'aux', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }, ]; const original = structuredClone({ tools, choice }); const inv = makeInvocation({ @@ -6636,10 +6636,10 @@ test.each(undeclaredSelectorChoices)('helper allocation does not capture an unde payload: { tools, tool_choice: choice }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { const helper = inv.payload.tools?.[0]; assert(helper?.type === 'function'); - assertEquals(helper.name, `${SHIM_TOOL_NAME}_3`); + assertEquals(helper.name, `${FUNCTION_TOOL_NAME}_3`); assertEquals(inv.payload.tool_choice, choice); return await script.run(); }); @@ -6655,23 +6655,23 @@ test.each([ const records: DispatchRecord[] = []; const iter = consumeTurnStreaming(framesOf( mkResponseCreated(), - eventFrame({ type: 'response.output_item.added', output_index: 0, item: { type: 'function_call', name: SHIM_TOOL_NAME, id: 'item', call_id: 'call', arguments: '', status: 'in_progress' } }), - eventFrame({ type: 'response.output_item.done', output_index: 0, item: { type: 'function_call', name: SHIM_TOOL_NAME, id: 'item', call_id: 'call', arguments: '{}', status: 'completed', ...changed } }), + eventFrame({ type: 'response.output_item.added', output_index: 0, item: { type: 'function_call', name: FUNCTION_TOOL_NAME, id: 'item', call_id: 'call', arguments: '', status: 'in_progress' } }), + eventFrame({ type: 'response.output_item.done', output_index: 0, item: { type: 'function_call', name: FUNCTION_TOOL_NAME, id: 'item', call_id: 'call', arguments: '{}', status: 'completed', ...changed } }), mkResponseCompleted(), - ), createMergeState(), true, new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []); + ), createMergeState(), true, new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []); await assertRejects(() => drain(iter, records), Error, 'changed a hosted-tool function identity'); assertEquals(records, []); }); test.each([undefined, 'item'])('hosted dispatch accepts a matching call instance with item ID %s', async id => { const records: DispatchRecord[] = []; - const item = { type: 'function_call' as const, name: SHIM_TOOL_NAME, id, call_id: 'call', arguments: '{"q":"matched"}', status: 'completed' as const }; + const item = { type: 'function_call' as const, name: FUNCTION_TOOL_NAME, id, call_id: 'call', arguments: '{"q":"matched"}', status: 'completed' as const }; const iter = consumeTurnStreaming(framesOf( mkResponseCreated(), eventFrame({ type: 'response.output_item.added', output_index: 0, item: { ...item, arguments: '', status: 'in_progress' } }), eventFrame({ type: 'response.output_item.done', output_index: 0, item }), mkResponseCompleted(), - ), createMergeState(), true, new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []); + ), createMergeState(), true, new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []); await drain(iter, records); assertEquals(records.length, 1); }); @@ -6681,16 +6681,16 @@ for (const targetApi of ['openaiChatCompletions', 'anthropicMessages', 'openaiRe const { backend } = makeStubDeps(); const inv = makeInvocation({ targetApi, enabledFlags: new Set(['openai-responses-web-search-shim']), payload: { - tools: [{ type: 'web_search' }, { type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object' } }] }], + tools: [{ type: 'web_search' }, { type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object' } }] }], }, }); const ctx = makeGatewayCtx(); let turns = 0; - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, ctx, async () => { + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, ctx, async () => { turns++; - assertEquals(inv.payload.tools?.[0], { ...inv.payload.tools?.[0], name: `${SHIM_TOOL_NAME}_2` }); + assertEquals(inv.payload.tools?.[0], { ...inv.payload.tools?.[0], name: `${FUNCTION_TOOL_NAME}_2` }); if (targetApi === 'openaiResponses') { - const item = { type: 'function_call' as const, name: SHIM_TOOL_NAME, namespace: 'functions', call_id: 'client', arguments: '{}', status: 'completed' as const }; + const item = { type: 'function_call' as const, name: FUNCTION_TOOL_NAME, namespace: 'functions', call_id: 'client', arguments: '{}', status: 'completed' as const }; return await scriptedRun([[ mkResponseCreated(), eventFrame({ type: 'response.output_item.added', output_index: 0, item: { ...item, status: 'in_progress' } }), @@ -6723,7 +6723,7 @@ for (const targetApi of ['openaiChatCompletions', 'anthropicMessages', 'openaiRe }); assertEquals(turns, 1); assertEquals(backend.calls, []); - assertEquals(findResponseCompleted(frames).response.output.map(item => item.type === 'function_call' ? [item.type, item.name, item.namespace] : [item.type]), [['function_call', SHIM_TOOL_NAME, 'functions']]); + assertEquals(findResponseCompleted(frames).response.output.map(item => item.type === 'function_call' ? [item.type, item.name, item.namespace] : [item.type]), [['function_call', FUNCTION_TOOL_NAME, 'functions']]); }); } @@ -6731,13 +6731,13 @@ test('native helper injection allocates past namespace children while retaining const { backend } = makeStubDeps(); const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']), payload: { - tools: [{ type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }, { type: 'web_search' }], + tools: [{ type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }, { type: 'web_search' }], }, }); - const script = scriptedRun([fcTurn(0, 'hosted', `${SHIM_TOOL_NAME}_2`, '{"search_query":[{"q":"hosted"}]}'), messageTurn('done')]); - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); + const script = scriptedRun([fcTurn(0, 'hosted', `${FUNCTION_TOOL_NAME}_2`, '{"search_query":[{"q":"hosted"}]}'), messageTurn('done')]); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.[1].type, 'function'); - assertEquals((inv.payload.tools?.[1] as { name: string }).name, `${SHIM_TOOL_NAME}_2`); + assertEquals((inv.payload.tools?.[1] as { name: string }).name, `${FUNCTION_TOOL_NAME}_2`); assertEquals(backend.calls.length, 1); assertEquals(script.callCount(), 2); assertEquals(findResponseCompleted(frames).response.status, 'completed'); @@ -6747,20 +6747,20 @@ for (const target of ['openaiChatCompletions', 'anthropicMessages'] as const) { for (const mode of ['forced', 'auto', 'required'] as const) { test(`${target} ${mode} helper choice follows a namespace-induced alias and restores its original echo`, async () => { const { backend } = makeStubDeps(); - const selector = { type: 'function' as const, name: SHIM_TOOL_NAME }; + const selector = { type: 'function' as const, name: FUNCTION_TOOL_NAME }; const choice: OpenAIResponsesToolChoice = mode === 'forced' ? selector : { type: 'allowed_tools', mode, tools: [selector] }; const original = structuredClone(choice); const inv = makeInvocation({ targetApi: target, payload: { - tools: [{ type: 'web_search' }, { type: 'namespace', name: 'client', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }], + tools: [{ type: 'web_search' }, { type: 'namespace', name: 'client', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }], tool_choice: choice, }, }); - const alias = `${SHIM_TOOL_NAME}_2`; + const alias = `${FUNCTION_TOOL_NAME}_2`; const script = scriptedRun([fcTurn(0, 'hosted', alias, '{"search_query":[{"q":"hosted"}]}'), messageTurn('done')]); const choices: unknown[] = []; - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { choices.push(structuredClone(inv.payload.tool_choice)); const before = structuredClone(inv.payload); const trip = target === 'openaiChatCompletions' @@ -6790,27 +6790,27 @@ for (const owner of ['function history', 'custom history', 'additional_tools', ' const { backend } = makeStubDeps(); const tools: OpenAIResponsesTool[] = [ { type: 'web_search' }, - { type: 'namespace', name: 'client', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }, + { type: 'namespace', name: 'client', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }, ]; const input: OpenAIResponsesInputItem[] = [{ type: 'message', role: 'user', content: 'Continue.' }]; if (owner === 'function history') { - input.push({ type: 'function_call', name: SHIM_TOOL_NAME, call_id: 'past', arguments: '{}', status: 'completed' }); + input.push({ type: 'function_call', name: FUNCTION_TOOL_NAME, call_id: 'past', arguments: '{}', status: 'completed' }); } else if (owner === 'custom history') { - input.push({ type: 'custom_tool_call', name: SHIM_TOOL_NAME, call_id: 'past', input: 'client input' }); + input.push({ type: 'custom_tool_call', name: FUNCTION_TOOL_NAME, call_id: 'past', input: 'client input' }); } else if (owner === 'additional_tools') { - input.push({ type: 'additional_tools', role: 'developer', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }); + input.push({ type: 'additional_tools', role: 'developer', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }); } else { - input.push({ type: 'tool_search_output', tools: [{ type: 'custom', name: SHIM_TOOL_NAME }] }); + input.push({ type: 'tool_search_output', tools: [{ type: 'custom', name: FUNCTION_TOOL_NAME }] }); } - const selector = { type: 'function' as const, name: SHIM_TOOL_NAME }; + const selector = { type: 'function' as const, name: FUNCTION_TOOL_NAME }; const choice: OpenAIResponsesToolChoice = selection === 'forced' ? selector : { type: 'allowed_tools', mode: 'auto', tools: [selector] }; const inv = makeInvocation({ payload: { tools, input, tool_choice: choice } }); - const script = scriptedRun([fcTurn(0, 'client', SHIM_TOOL_NAME, '{}')]); - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { + const script = scriptedRun([fcTurn(0, 'client', FUNCTION_TOOL_NAME, '{}')]); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { assertEquals(inv.payload.tool_choice, choice); const helper = inv.payload.tools?.[0]; assert(helper?.type === 'function'); - assertEquals(helper.name, `${SHIM_TOOL_NAME}_2`); + assertEquals(helper.name, `${FUNCTION_TOOL_NAME}_2`); return await script.run(); }); assertEquals(script.callCount(), 1); @@ -6826,7 +6826,7 @@ for (const selectorType of ['web_search', 'function'] as const) { const { backend } = makeStubDeps(); const choice: OpenAIResponsesToolChoice = { type: 'allowed_tools', mode: 'required', tools: [ - { type: selectorType, ...(selectorType === 'function' ? { name: SHIM_TOOL_NAME } : {}) }, + { type: selectorType, ...(selectorType === 'function' ? { name: FUNCTION_TOOL_NAME } : {}) }, ...(includeClient ? [{ type: 'function', name: 'client_allowed' }] : []), ], }; @@ -6835,14 +6835,14 @@ for (const selectorType of ['web_search', 'function'] as const) { const script = scriptedRun([searchCallTurn(0, 'hosted', 'query'), messageTurn('done')]); const choices: unknown[] = []; const declarations: Array = []; - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { choices.push(inv.payload.tool_choice); declarations.push(inv.payload.tools); return await script.run(); }); - const allowed = [{ type: 'function', name: SHIM_TOOL_NAME }, ...(includeClient ? [{ type: 'function', name: 'client_allowed' }] : [])]; + const allowed = [{ type: 'function', name: FUNCTION_TOOL_NAME }, ...(includeClient ? [{ type: 'function', name: 'client_allowed' }] : [])]; assertEquals(choices, [{ type: 'allowed_tools', mode: 'required', tools: allowed }, { type: 'allowed_tools', mode: 'auto', tools: allowed }]); - assertEquals(declarations[0]?.map(tool => 'name' in tool ? tool.name : undefined), [SHIM_TOOL_NAME, 'client_allowed', 'excluded']); + assertEquals(declarations[0]?.map(tool => 'name' in tool ? tool.name : undefined), [FUNCTION_TOOL_NAME, 'client_allowed', 'excluded']); assert(declarations[0] === declarations[1], 'mode demotion must retain the same declaration inventory'); assertEquals(script.callCount(), 2); assertEquals(backend.calls.length, 1); @@ -6860,7 +6860,7 @@ test('required client-only allowed_tools is not relaxed by an unrelated hosted i // allowed subset, so the test observes that client policy is not demoted. const script = scriptedRun([searchCallTurn(0, 'hosted', 'query'), messageTurn('done')]); const choices: unknown[] = []; - await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { choices.push(structuredClone(inv.payload.tool_choice)); return await script.run(); }); @@ -6874,12 +6874,12 @@ test('helper echoes distinguish namespaced functions and rewrite hosted allowed_ makeStubDeps(); const choice: OpenAIResponsesToolChoice = { type: 'allowed_tools', mode: 'required', tools: [{ type: 'web_search' }] }; const inv = makeInvocation({ payload: { tool_choice: structuredClone(choice) } }); - const qualified = { type: 'function' as const, name: SHIM_TOOL_NAME, namespace: 'client' }; - const { frames } = await runShimAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { - assertEquals(inv.payload.tool_choice, { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }); + const qualified = { type: 'function' as const, name: FUNCTION_TOOL_NAME, namespace: 'client' }; + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { + assertEquals(inv.payload.tool_choice, { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }); return await scriptedRun([[ mkResponseCreated(), - eventFrame({ type: 'response.completed', response: { ...emptyResult('upstream', 'completed'), tools: [qualified, { type: 'function', name: SHIM_TOOL_NAME }], tool_choice: inv.payload.tool_choice } }), + eventFrame({ type: 'response.completed', response: { ...emptyResult('upstream', 'completed'), tools: [qualified, { type: 'function', name: FUNCTION_TOOL_NAME }], tool_choice: inv.payload.tool_choice } }), ]]).run(); }); const tools = findResponseCompleted(frames).response.tools; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/wire.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/wire.ts index 173ca4ff03..a081c8f508 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/wire.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/wire.ts @@ -22,7 +22,7 @@ import type { Stage } from '@floway-dev/pipeline'; * this wire reports; the vendor dialects are how one upstream spells both. All of them apply * to whatever body this wire actually sends and to nothing that leaves for another protocol. * - * The order is the one the rules had as an onion, which is the same order in both directions: + * Requests descend through this array and responses return through it in reverse: * a stage earlier in the array rewrites the request first and reads the answer last. So the * usage chunk is asked for above everything, and coming back the vendor dialects have the * first say — the generic rules above them then read a body already in OpenAI-canonical form, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/begin-stored-attempt.ts b/packages/gateway/src/data-plane/chat/openai-responses/begin-stored-attempt.ts index e520836391..efb706413d 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/begin-stored-attempt.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/begin-stored-attempt.ts @@ -3,13 +3,9 @@ import type { ChatServices } from '../services.ts'; import { defineStage } from '@floway-dev/pipeline'; /** - * Reseeds the store's per-attempt scratchpad from what the membrane hydrated. - * - * Below the fork because it is per attempt: what one attempt wrote into the scratchpad is - * not what the next one starts from, and re-running the suffix is what clears it. Nothing in - * a pipelined turn writes to it — the hosted-tool shim is what writes, and it does not run at - * all — so today this only puts back the state the stored rows already carried, which is what - * lets an item this turn re-emits be stored with it intact. + * Each candidate starts with the hydrated private payloads. Hosted tools can add execution + * state during that attempt; reseeding here prevents a failed candidate's state from leaking + * into failover while preserving payloads when stored items are replayed. */ export const beginStoredAttempt = defineStage< Fields<'request.chat.openaiResponses.privatePayloads'>, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts index 0f6a479a8a..4c3bf83a50 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts @@ -175,7 +175,7 @@ export const hostedTools = ( }, }, // A tool declaration this gateway cannot accept is answered here rather than dialled: the - // upstream would have been asked for a tool it does not implement, on a body the shim wrote. + // upstream would have been asked for a tool it does not implement, on a body the dispatcher wrote. return: { provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers'], }, @@ -272,7 +272,7 @@ export const hostedTools = ( }, }); -/** One descent, asked with the payload the shim wrote. */ +/** One descent, asked with the payload the dispatcher wrote. */ const descend = async ( next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>) => Promise, facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, @@ -336,16 +336,16 @@ async function* spliceTurns(args: { merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); for (;;) { const turn = currentTurn; - const executedShim = turn.dispatched.length > 0; + const executedHostedTools = turn.dispatched.length > 0; if (turn.terminalStatus.kind === 'failed') { failed = true; - if (executedShim) yield* materializeHostedToolItems(turn.dispatched, merge, store); + if (executedHostedTools) yield* materializeHostedToolItems(turn.dispatched, merge, store); yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: turn.terminalStatus.response.error }, active); return; } if (turn.terminalStatus.kind === 'incomplete') { - if (executedShim) yield* materializeHostedToolItems(turn.dispatched, merge, store); + if (executedHostedTools) yield* materializeHostedToolItems(turn.dispatched, merge, store); yield synthesizeTerminalEnvelope(merge, { kind: 'incomplete', incompleteDetails: turn.terminalStatus.response.incomplete_details }, active); return; } @@ -357,7 +357,7 @@ async function* spliceTurns(args: { }, active); return; } - if (!executedShim && !turn.sawClientToolCall) { + if (!executedHostedTools && !turn.sawClientToolCall) { yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); return; } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts index 95d42b208c..9942abd480 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts @@ -27,7 +27,7 @@ import { } from '@floway-dev/protocols/openai-responses'; import { providerModelOf, type Fetcher, type OpenAIImagesEditsRequest, type Provider, type ModelCandidate, type ProviderModel } from '@floway-dev/provider'; -export const SHIM_TOOL_NAME = 'image_generation'; +export const FUNCTION_TOOL_NAME = 'image_generation'; // Default image backend when the hosted tool omits `model`. gpt-image-2 is // the reference backend Azure's native OpenAI Responses `image_generation` routes @@ -35,7 +35,7 @@ export const SHIM_TOOL_NAME = 'image_generation'; export const DEFAULT_IMAGE_MODEL = 'gpt-image-2'; // Safety valve on the multi-turn ReAct loop: cap how many real image backend -// calls one response may dispatch (counted on `ShimState.imageDispatchCount`, +// calls one response may dispatch (counted on `HostedToolState.imageDispatchCount`, // not the shared ReAct turn count, so unrelated turns do not consume the // budget). Past the cap the dispatcher replays an exhausted-budget tool output // instead of hitting the backend, so a model that keeps retrying after failures @@ -44,7 +44,7 @@ const IMAGE_ITERATION_CAP = 10; // Public OpenAI Responses `image_generation` tool config enums (Azure-strict // surface). `webp` and arbitrary `WxH` sizes are rejected because the -// native Azure path rejects them; the shim mirrors that vocabulary rather +// native Azure path rejects them; the dispatcher mirrors that vocabulary rather // than passing them to a backend that would 400 with a different shape. const ALLOWED_SIZES = new Set(['1024x1024', '1024x1536', '1536x1024', 'auto']); const ALLOWED_QUALITIES = new Set(['low', 'medium', 'high', 'auto']); @@ -57,7 +57,7 @@ const ALLOWED_INPUT_FIDELITY = new Set(['high', 'low']); // gpt-image-* `/images/edits` accepts only these input image mimetypes; a live // Azure probe confirmed png/jpeg/webp succeed while gif is rejected with // `unsupported_file_mimetype`. Native OpenAI Responses accepts the same GIF and -// re-encodes it before editing, so the shim mirrors that behavior through the +// re-encodes it before editing, so the dispatcher mirrors that behavior through the // platform image processor. Common aliases are folded onto the backend form. type EditMime = 'image/png' | 'image/jpeg' | 'image/webp'; @@ -79,7 +79,7 @@ const editFileExt = (mime: EditMime): string => mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; // The public `image_generation` tool-config surface. Azure rejects any other -// field with `unknown_parameter`, so the shim mirrors that strictness rather +// field with `unknown_parameter`, so the dispatcher mirrors that strictness rather // than silently forwarding unknown fields (which would diverge from the // emulated surface and hide client bugs). `n` is deliberately absent: Azure // echoes `n:1` internally but rejects a client-supplied `tools[].n`. @@ -92,7 +92,7 @@ const KNOWN_TOOL_FIELDS = new Set([ export const isHostedImageGenerationTool = (tool: OpenAIResponsesTool): tool is OpenAIResponsesHostedTool => tool.type === 'image_generation'; -// Identity canonicalization for image_generation: the shim doesn't +// Identity canonicalization for image_generation: the dispatcher doesn't // depend on filled defaults to run, and the OpenAI spec defaults for // `background` / `quality` / `size` / etc. observed via Azure echo // (all `'auto'`) signal "backend decides" rather than concrete values @@ -242,9 +242,9 @@ const decodeInputImageDataUrl = ( } }; -// The orchestrator-visible tool config the shim layers onto the backend +// The orchestrator-visible tool config the dispatcher layers onto the backend // call. Mirrors Azure: the orchestrator only chooses `prompt`; everything -// here is read from the client's hosted-tool entry and applied by the shim. +// here is read from the client's hosted-tool entry and applied by the dispatcher. export interface ImageGenerationConfig { model: string; size?: string; @@ -322,7 +322,7 @@ const integerInRange = (value: unknown, param: string, min: number, max: number) }; // Validate one hosted `image_generation` declaration against the fields and -// values supported by this shim. The caller validates every +// values supported by this dispatcher. The caller validates every // declaration before selecting the last config, so an earlier invalid entry // fails. toolPath identifies its source carrier in the error. // https://developers.openai.com/api/docs/guides/tools-image-generation @@ -463,7 +463,7 @@ export const prepareImageGenerationConfig = ( }; // Single optional `prompt` parameter — matches the native `image_gen.imagegen` -// tool's surface (size/quality/etc. are NOT model-chosen; the shim layers them +// tool's surface (size/quality/etc. are NOT model-chosen; the dispatcher layers them // on from the client config, exactly like Azure). A minimal description // elicits native-quality refined prompts while costing ~50 input tokens vs // the native hosted tool's ~2300. @@ -901,7 +901,7 @@ const errorFromBody = (body: string, status: number): { type?: string; code: str // from the payload the current descent is sending, so an image generated in an // earlier turn (fed back as an `input_image`) becomes editable in a later one. `imageDispatchCount` bounds how many real backend image // calls one response may issue. -interface ShimState { +interface HostedToolState { config: MaterializedImageGenerationConfig; apiKeyId: string; upstreamIds: readonly string[] | null; @@ -989,7 +989,7 @@ const serverError = (e: unknown): ImageError => ({ // an `ImageError` so the caller always produces a terminal image item. const resolveImageCandidate = async ( isEdit: boolean, - state: ShimState, + state: HostedToolState, ): Promise<{ ok: true; candidate: ModelCandidate } | { ok: false; error: ImageError }> => { const endpointKey = isEdit ? 'openaiImagesEdits' : 'openaiImagesGenerations'; const endpointPath = isEdit ? '/images/edits' : '/images/generations'; @@ -1084,7 +1084,7 @@ const issueImageCall = async ( prompt: string, editRequest: OpenAIImagesEditsRequest | null, config: ImageGenerationConfig, - state: ShimState, + state: HostedToolState, stream: boolean, attempt: AttemptState, ): Promise<{ response: Response; modelKey: string }> => { @@ -1123,7 +1123,7 @@ const consumeImageResponse = async ( model: ProviderModel, modelKey: string, response: Response, - state: ShimState, + state: HostedToolState, billed: BillableEntity[], ): Promise => { const text = await response.text(); @@ -1245,7 +1245,7 @@ const streamImageGeneration = ( action: 'generate' | 'edit', isEdit: boolean, sources: readonly ImageSource[], - state: ShimState, + state: HostedToolState, settle: SettleImageCall, billed: BillableEntity[], ) => async function* (): AsyncGenerator { @@ -1426,7 +1426,7 @@ export const imageGenerationHostedTool: HostedToolRegistration = async (invocati // the upstream can read them, but there is no hosted tool to dispatch. return { type: 'active', - baseToolName: SHIM_TOOL_NAME, + baseToolName: FUNCTION_TOOL_NAME, transformItems: transformInputItemsForImageGeneration, }; } @@ -1494,7 +1494,7 @@ export const imageGenerationHostedTool: HostedToolRegistration = async (invocati ...(mask === undefined ? {} : { mask }), }; - const state: ShimState = { + const state: HostedToolState = { config: materializedConfig, gateway: gatewayCtx, apiKeyId: gatewayCtx.apiKeyId, @@ -1507,7 +1507,7 @@ export const imageGenerationHostedTool: HostedToolRegistration = async (invocati return { type: 'active', - baseToolName: SHIM_TOOL_NAME, + baseToolName: FUNCTION_TOOL_NAME, transformItems: transformInputItemsForImageGeneration, hosted: { hostedTypes: ['image_generation'], diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts index 6b41af4937..1310484fb3 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts @@ -8,7 +8,7 @@ import type { GatewayCtx, AttemptState } from '../../../shared/gateway-ctx.ts'; import { run, move } from '@floway-dev/pipeline'; /** - * The run a shim call is. + * The run a dispatcher call is. * * Its context is the parent's, with the three things a separate run owes itself: a start time of * its own, an attempt slot of its own — so the outer turn's upstream stamp survives — and a diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts index 144cb4f2c3..f2e1aefd3e 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts @@ -6,7 +6,7 @@ import type { WebSearchCallIR } from '../../../tools/web-search/operations.ts'; import { run, move } from '@floway-dev/pipeline'; /** - * The run a shim's search is. + * The run a dispatcher's search is. * * Its context is the parent's, with the three things a separate run owes itself: a start time of * its own, an attempt slot of its own — so the outer turn's upstream stamp survives — and a diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts index 8023c72b85..ffcb700012 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts @@ -12,7 +12,7 @@ export const createMergeState = (): MergeState => ({ accumulatedOutput: new Map(), accumulatedUsage: {}, lastSeenModel: null, - synthesizedResponseId: `resp_shim_${crypto.randomUUID().replace(/-/g, '')}`, + synthesizedResponseId: `resp_hosted_${crypto.randomUUID().replace(/-/g, '')}`, upstreamResponseSnapshot: undefined, }); @@ -123,7 +123,7 @@ export const isForcedHostedToolChoice = ( && typeof selector.name === 'string' && dispatchers.has(selector.name)); }; -// The shim demotes forced choice to `auto` after the first turn, so synthesized +// The dispatcher demotes forced choice to `auto` after the first turn, so synthesized // echoes restore the captured client shape rather than the final upstream echo. const restoreEchoedToolChoice = ( toolChoice: OpenAIResponsesToolChoice | null | undefined, @@ -215,7 +215,7 @@ export const historicalClientCallableUsesName = (name: string, input: readonly O // Collapse matching hosted declarations within each tools array. Keep the // last declaration's configuration at the first matching slot so unrelated // tools retain their relative order. -export const rewriteToolsForHostedShim = ( +export const rewriteHostedTools = ( tools: readonly OpenAIResponsesTool[], hosted: HostedToolRewrite, toolName: string, @@ -250,7 +250,7 @@ export const rewriteHostedDeclarations = ( let canonicalHostedTool: OpenAIResponsesHostedTool | undefined; const rewrite = (tools: OpenAIResponsesTool[]): OpenAIResponsesTool[] => { if (!tools.some(tool => hosted.canonicalize(tool) !== undefined)) return tools; - const result = rewriteToolsForHostedShim(tools, hosted, toolName); + const result = rewriteHostedTools(tools, hosted, toolName); canonicalHostedTool = result.canonicalHostedTool; return result.rewritten; }; @@ -292,7 +292,7 @@ const syntheticPrologueResponse = ( status: 'queued' | 'in_progress', ): OpenAIResponsesResult => { if (state.upstreamResponseSnapshot === undefined) { - throw new Error('Server-tool shim cannot synthesize an OpenAI Responses prologue envelope before an upstream response snapshot is captured.'); + throw new Error('Hosted-tool execution cannot synthesize an OpenAI Responses prologue envelope before an upstream response snapshot is captured.'); } const snapshot = state.upstreamResponseSnapshot; const restoredTools = restoreEchoedTools(snapshot.tools, active); @@ -441,7 +441,7 @@ export const consumeTurnStreaming = async function* ( const ensureModel = (): string => { if (merge.lastSeenModel === null) { - throw new Error('Server-tool shim cannot synthesize an OpenAI Responses envelope because upstream `response.created` did not report a `model` field.'); + throw new Error('Hosted-tool execution cannot synthesize an OpenAI Responses envelope because upstream `response.created` did not report a `model` field.'); } return merge.lastSeenModel; }; @@ -519,7 +519,7 @@ export const consumeTurnStreaming = async function* ( const item = event.item; if (item.type === 'function_call') { if (dispatchers.has(item.name)) { - // Reserve the downstream index the shim call occupies now, at + // Reserve the downstream index the dispatcher call occupies now, at // `.added`; the actual slot count is only known at `.done`, // where slot 0 takes this reserved index and any further slots // take fresh ones. Those stay contiguous because OpenAI Responses @@ -711,7 +711,7 @@ export const consumeTurnStreaming = async function* ( output: [], status: 'failed', error: { - message: `Upstream emitted ${priorLabel} without closing shim call items at upstream output_index ${unmatched.join(', ')}.`, + message: `Upstream emitted ${priorLabel} without closing dispatcher call items at upstream output_index ${unmatched.join(', ')}.`, code: 'server_error', }, incomplete_details: null, @@ -769,10 +769,10 @@ export const synthesizeTerminalEnvelope = ( active: readonly ActiveHostedTool[], ): ProtocolFrame => { if (state.lastSeenModel === null) { - throw new Error('Server-tool shim cannot synthesize an OpenAI Responses terminal envelope before upstream `response.created` reports a model.'); + throw new Error('Hosted-tool execution cannot synthesize an OpenAI Responses terminal envelope before upstream `response.created` reports a model.'); } if (state.upstreamResponseSnapshot === undefined) { - throw new Error('Server-tool shim cannot synthesize an OpenAI Responses terminal envelope before upstream `response.created` is captured.'); + throw new Error('Hosted-tool execution cannot synthesize an OpenAI Responses terminal envelope before upstream `response.created` is captured.'); } const output = materializeAccumulatedOutput(state); const usage = usageForWire(state); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/types.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/types.ts index e7442ff773..0b570a3997 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/types.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/types.ts @@ -124,7 +124,7 @@ export type ActiveHostedTool = Extract = new Set(WEB_SEARCH_HOSTED_TYPE_NAMES); -// Function-name regex `^[a-zA-Z0-9_-]+$` forbids dots, so the shim call +// Function-name regex `^[a-zA-Z0-9_-]+$` forbids dots, so the dispatcher call // uses the underscored form of the model's training-time `web.run`. -export const SHIM_TOOL_NAME = 'web_search'; +export const FUNCTION_TOOL_NAME = 'web_search'; // Put the selected hosted declaration's `user_location` in the replacement // function description so the model can use it as the default for local searches. @@ -60,7 +60,7 @@ const formatUserLocation = (loc: NonNullable): // local Tavily, Jina, and Microsoft Web IQ execution rejects the fields it // cannot implement before dispatch. // https://github.com/openai/codex/blob/2f19a57704fb7b1db032bc38cf995034254eaebb/codex-rs/codex-api/src/search.rs#L31-L213 -export const buildShimFunctionTool = ( +export const buildFunctionTool = ( canonical: OpenAIResponsesHostedTool, name: string, ): OpenAIResponsesFunctionTool => { @@ -332,8 +332,8 @@ const validateDomainListEntry = ( return { ok: true }; }; -// Validate the hosted web-search fields the shim interprets. The model-bound -// replacement uses the shim command schema; backend settings such as +// Validate the hosted web-search fields the dispatcher interprets. The model-bound +// replacement uses the dispatcher command schema; backend settings such as // `external_web_access` are handled separately, while unhandled fields such // as `return_token_budget` are omitted from that request. const validateHostedEntry = (tool: OpenAIResponsesHostedTool): PrepareToolsError | null => { @@ -381,7 +381,7 @@ const validateHostedEntry = (tool: OpenAIResponsesHostedTool): PrepareToolsError // Validate every hosted declaration before selecting the last one's runtime // filters, so an earlier invalid declaration cannot be masked. -export const prepareToolsForShim = ( +export const prepareTools = ( tools: OpenAIResponsesTool[], paths?: readonly string[], ): PrepareToolsResult => { @@ -411,9 +411,9 @@ export const prepareToolsForShim = ( const MAX_MALFORMED_WIRE_DUMP_CHARS = 1024; /** - * Persistent `payload.private` shape for one `web_search_call`. One shim call + * Persistent `payload.private` shape for one `web_search_call`. One dispatcher call * function_call corresponds to exactly one wsc and one op — multi-op - * shim calls (multi-kind mix or multi-instance same-kind) are rejected at + * dispatcher calls (multi-kind mix or multi-instance same-kind) are rejected at * dispatch with an `ambiguous` error, so there is never an array to * denormalize. The persisted-payload key IS the wsc id, so we don't repeat * it inside. @@ -425,13 +425,13 @@ const MAX_MALFORMED_WIRE_DUMP_CHARS = 1024; * so the upstream model's prior assistant turn looks bit-exact. * * - `ir` stores the action, structured results, and optional upstream - * model-facing output straight from `planShimSlots`. Replay uses + * model-facing output straight from `planDispatchSlots`. Replay uses * `renderWebSearchCallOutput`, which preserves that output when present * and otherwise renders the action and results. * * Version-tagged: an unknown `v` falls through the no-payload branch in * `transformInputItemsForWebSearch` (action re-serialized into the - * shim call shape, output replaced with the not-preserved notice). Starts + * dispatcher call shape, output replaced with the not-preserved notice). Starts * at 1; bump only on a wire-incompatible change after release. */ export interface WebSearchCallPrivatePayload { @@ -460,12 +460,12 @@ export const synthesizeWebSearchCallId = (): string => createRandomOpenAIRespons // replay call_id never reads as a web-search item id in logs. const synthesizeReplayCallId = (): string => shortId('cc_replay'); -// Re-serializes a wire `action` back into the shim's JSON arguments +// Re-serializes a wire `action` back into the dispatcher's JSON arguments // shape (`{search_query:[{q}]}` / `{open:[{ref_id}]}` / // `{find:[{ref_id,pattern}]}`). Used only on the replay-fallback path to // fill the paired function_call's `arguments` when no private payload // exists; the happy path replays the upstream's original args verbatim. -const actionToShimCallArgsJson = (action: OpenAIResponsesWebSearchAction): string => { +const actionToFunctionCallArgsJson = (action: OpenAIResponsesWebSearchAction): string => { switch (action.type) { case 'search': return JSON.stringify({ @@ -497,9 +497,9 @@ const actionToShimCallArgsJson = (action: OpenAIResponsesWebSearchAction): strin // 2. No payload (`store: false`, expired, foreign id, cross-account, or // schema-version mismatch): degrade to a synthesized pair whose // `function_call.arguments` is the wire action re-serialized into -// the shim call shape (so the model still sees what it asked for) and +// the dispatcher call shape (so the model still sees what it asked for) and // whose `function_call_output` text is the not-preserved placeholder. -// The shim deliberately does not read `item.results` from the +// The dispatcher deliberately does not read `item.results` from the // wire — turn 1's wire results may or may not exist depending on the // client's `include` opt-in, and trusting them across the wire would // couple state correctness to client storage discipline. @@ -563,7 +563,7 @@ export const transformInputItemsForWebSearch = ( type: 'function_call', call_id: callId, name: toolName, - arguments: actionToShimCallArgsJson(item.action), + arguments: actionToFunctionCallArgsJson(item.action), status: 'completed', }, { @@ -578,12 +578,12 @@ export const transformInputItemsForWebSearch = ( return out; }; -// The shim's execution session plus the one wire-shaping flag that lives +// The dispatcher's execution session plus the one wire-shaping flag that lives // only on the OpenAI Responses side. -interface ShimState extends WebSearchExecutionSession { +interface HostedToolState extends WebSearchExecutionSession { // Set when the client passed `include: ["web_search_call.results"]` on // the request. Native OpenAI Responses gates the `results` field on this - // include token; the shim follows suit on the wire item — but the IR + // include token; the dispatcher follows suit on the wire item — but the IR // (and therefore `payload.private`) always carries the real results // so a subsequent turn echoing the item id can be hydrated regardless. includeSearchResults: boolean; @@ -604,11 +604,11 @@ const eagerResolver = (promise: Promise): (() => Promise) => { }; }; -const planShimSlots = ( +const planDispatchSlots = ( parsed: ParsedWebSearchOperations, commands: Record, toolName: string, - state: ShimState, + state: HostedToolState, loopState: HostedToolLoopState, ): { id: string; resolve: () => Promise } => { if (loopState.iterationCount > ITERATION_CAP) { @@ -626,7 +626,7 @@ const planShimSlots = ( return { id: synthesizeWebSearchCallId(), resolve: async () => schemaErrorIr( - 'malformed shim call arguments', + 'malformed dispatcher call arguments', 'Malformed arguments', 'Error: arguments must be a JSON object with sub-property arrays (search_query[], open[], find[]).', ), @@ -691,7 +691,7 @@ const planShimSlots = ( return { id: synthesizeWebSearchCallId(), resolve: async () => schemaErrorIr( - 'ambiguous shim call', + 'ambiguous dispatcher call', 'Ambiguous tool call', `Error: ambiguous \`${toolName}\` tool call — each function_call maps to one web_search_call. ` + 'Multiple `search_query` entries are fine (they collapse into one search). ' @@ -717,7 +717,7 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga const hasReplayInput = invocation.payload.input.some(i => i.type === 'web_search_call'); if (!hasHostedWebSearch && !hasReplayInput) return { type: 'inactive' }; - const prepared = prepareToolsForShim(tools, declarations.map(entry => entry.path)); + const prepared = prepareTools(tools, declarations.map(entry => entry.path)); if (!prepared.ok) { return { type: 'invalid-request', @@ -732,7 +732,7 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga let configuredProvider: Promise | undefined; const hosted = tools.filter(isHostedWebSearchTool).at(-1); const settings = alphaSearchSettingsFromHosted(hosted); - const state: ShimState = { + const state: HostedToolState = { filters, pageCache: new Map(), getProvider: () => { @@ -765,17 +765,17 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga return { type: 'active', - baseToolName: SHIM_TOOL_NAME, + baseToolName: FUNCTION_TOOL_NAME, transformItems: (items, toolName) => transformInputItemsForWebSearch(items, toolName, id => gatewayCtx.store.getPrivatePayload(id)), ...(hasHostedWebSearch ? { hosted: { hostedTypes: WEB_SEARCH_HOSTED_TYPE_NAMES, canonicalize: canonicalizeWebSearchTool, - buildFunctionTool: buildShimFunctionTool, + buildFunctionTool, dispatcher: ({ intercepted, loopState }) => { const commands = intercepted.arguments ?? {}; - const slot = planShimSlots(parseWebSearchOperations(intercepted.arguments), commands, intercepted.name, state, loopState); + const slot = planDispatchSlots(parseWebSearchOperations(intercepted.arguments), commands, intercepted.name, state, loopState); const functionCallItem: OpenAIResponsesFunctionToolCallItem = { type: 'function_call', call_id: intercepted.callId, From 3b35ff5919a0a34db0e1aadf3e67941b456bd514 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 03:12:13 +0800 Subject: [PATCH 113/156] fix(responses): record client protocol failures before framing Record the semantic terminal and synthesized failure frames before HTTP framing, and combine the client completion verdict with measured upstream usage. Preserve WebSocket error propagation and failed snapshot diagnostics. Exercise failures before response creation, during the upstream stream, and after the upstream usage terminal while snapshot persistence fails. Compare the selected recorded client stream with the actual SSE events. --- .../chat/openai-responses/emit_test.ts | 126 ++++++++++++++++++ .../data-plane/chat/openai-responses/emit.ts | 96 +++++++------ 2 files changed, 182 insertions(+), 40 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts new file mode 100644 index 0000000000..f35d10f834 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts @@ -0,0 +1,126 @@ +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; + +import { emitOpenAIResponses } from '../../../../src/data-plane/chat/openai-responses/emit.ts'; +import type { Fields } from '../../../../src/data-plane/chat/openai-responses/facts.ts'; +import { LayeredOpenAIResponsesStatefulStore, MemoryOpenAIResponsesStatefulBacking } from '../../../../src/data-plane/chat/openai-responses/items/store.ts'; +import { meterUsage } from '../../../../src/data-plane/chat/openai-responses/meter.ts'; +import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; +import { openRunDump } from '../../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../../src/repo/index.ts'; +import type { ApiKey } from '../../../../src/repo/types.ts'; +import { installDumpStubs } from '../../../dump/test-fixtures.ts'; +import { flushBackground, trackBackground } from '../../../test-utils/background-tracker.ts'; +import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; +import { compose, createRunReader, defineStage, move, run, type DumpEvent } from '@floway-dev/pipeline'; +import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; +import type { ClientOpenAIResponsesStreamEvent, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { testTelemetryModelIdentity } from '@floway-dev/test-utils'; + +const key: ApiKey = { + id: 'key_emit', userId: 1, name: 'Emitter key', key: 'emitter-key', serverSecret: '11'.repeat(32), + createdAt: '2026-01-01T00:00:00.000Z', upstreamIds: null, deletedAt: null, + dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0, +}; + +type Entry = Fields<'ingress.chat.openaiResponses.wantsStream'>; +type Answer = Entry & Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'>; +type Exit = Fields<'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage'>; + +beforeEach(() => { + initRepo({ upstreams: { getById: async () => null } } as never); +}); +afterEach(() => { vi.restoreAllMocks(); }); + +for (const failureAt of ['before-response', 'upstream-stream', 'snapshot'] as const) { + test(`records the exact client protocol failure frames for ${failureAt} failure`, async () => { + const announced = failureAt !== 'before-response'; + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const timing = { upstreamCallStartedAt: 1, firstOutputTokenAt: null }; + const dump = openRunDump(key, { + method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null }, + }, trackBackground, true, timing); + if (dump === null) throw new Error('retained request did not open its recording'); + const fault = new Error('upstream connection broke', { cause: new Error('socket reset') }); + const backing = new MemoryOpenAIResponsesStatefulBacking(); + const store = failureAt === 'snapshot' ? new LayeredOpenAIResponsesStatefulStore({ apiKeyId: key.id, reads: [backing], writes: [backing] }) : undefined; + if (store !== undefined) { + expect(store.writesState).toBe(true); + vi.spyOn(store, 'commitSnapshot').mockRejectedValue(fault); + } + const gateway = mockChatGatewayCtx({ dump, wantsStream: true, backgroundScheduler: trackBackground, attempt: { timing, telemetry: undefined }, ...(store === undefined ? {} : { store }) }); + const upstream = (async function* (): AsyncIterable> { + if (announced) { + yield { + type: 'event', event: { + type: 'response.created', sequence_number: 0, + response: { id: 'upstream-response', object: 'response', model: 'model', status: 'in_progress', output: [], error: null, incomplete_details: null }, + }, + }; + } + if (failureAt === 'snapshot') { + yield { + type: 'event', event: { + type: 'response.completed', sequence_number: 1, + response: { id: 'upstream-response', object: 'response', model: 'model', status: 'completed', output: [], error: null, incomplete_details: null, usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 } }, + }, + }; + return; + } + throw fault; + })(); + const dial = defineStage({ + name: 'scriptedDial', + return: { provides: ['response.chat.openaiResponses', 'response.usage.billable', 'response.http.headers'] }, + execute: async facts => move({ + ...facts, 'response.chat.openaiResponses': { kind: 'stream', frames: upstream }, + 'response.usage.billable': [{ identity: testTelemetryModelIdentity, quantities: {} }], 'response.http.headers': [], + }), + }); + const outcome = await run(compose('clientFailure', [ + emitOpenAIResponses({ model: 'model', input: [], store: false }, 'sse'), + meterUsage('response.chat.openaiResponses.streamedUsage'), dial, + ]), move({ 'ingress.chat.openaiResponses.wantsStream': true }), { gateway, dump: dump.sink }); + const wire: SseFrame[] = []; + for await (const frame of outcome.facts['response.chat.openaiResponses.rendered'] as AsyncIterable) wire.push(frame); + expect((await outcome.facts['response.chat.openaiResponses.streamedUsage'])?.failed).toBe(true); + if (store !== undefined) { + expect(store.commitSnapshot).toHaveBeenCalledTimes(1); + expect((await outcome.facts['response.chat.openaiResponses.streamedUsage'])?.billable).toMatchObject([{ quantities: { input_tokens: '10', output_tokens: '5' } }]); + } + await outcome.drain(); + dump.finalize(200, 0); + await flushBackground(); + + const read = createRunReader(); + const streams = new Map[]>(); + let client: number | null = null; + const record = stubs.stored[0]?.record; + if (!record) throw new Error('completed request did not persist its recording'); + for (const line of new TextDecoder().decode(record.events).split('\n').filter(Boolean)) { + const event = JSON.parse(line) as DumpEvent; + const decoded = read(event); + if (decoded?.facts && 'response.chat.clientFrames' in decoded.facts) { + client = (decoded.facts['response.chat.clientFrames'] as { stream: number }).stream; + } + if (event.type === 'stream.frame') { + const frames = streams.get(event.streamId) ?? []; + frames.push(...decoded?.frames as ProtocolFrame[]); + streams.set(event.streamId, frames); + } + } + const frames = client === null ? [] : streams.get(client) ?? []; + expect(frames.length).toBeGreaterThan(0); + expect(frames.map(frame => frame.type === 'done' ? '[DONE]' : frame.event)).toEqual(wire.map(frame => JSON.parse(frame.data))); + const expectedEvents = failureAt === 'snapshot' + ? ['response.created', 'response.output_item.added', 'response.output_item.done', 'error', 'response.failed'] + : announced ? ['response.created', 'error', 'response.failed'] : ['error']; + expect(frames.filter(frame => frame.type === 'event').map(frame => frame.event.type)).toEqual(expectedEvents); + const error = frames.find(frame => frame.type === 'event' && frame.event.type === 'error'); + expect(error).toMatchObject({ event: { error: { message: fault.message, cause: { message: 'socket reset' } } } }); + if (announced) { + const created = frames[0] as ProtocolFrame>; + if (created.type !== 'event') throw new Error('announced response has no creation event'); + expect(frames.at(-1)).toMatchObject({ event: { response: { id: created.event.response.id, status: 'failed' } } }); + } + }); +} diff --git a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts index 29f67fc958..a1efca4723 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts @@ -1,12 +1,13 @@ import { wrapOpenAIResponsesClientEgress } from './client-output.ts'; import { internalErrorEnvelope } from './errors.ts'; -import type { OpenAIResponsesStreamFraming, Fields } from './facts.ts'; +import { OPENAI_RESPONSES_STREAMED_USAGE, type OpenAIResponsesStreamFraming, type Fields } from './facts.ts'; import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; -import { defineStage, move } from '@floway-dev/pipeline'; -import { doneFrame, sseFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { defer, defineStage, move, type Deferred } from '@floway-dev/pipeline'; +import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectOpenAIResponsesProtocolEventsToResult, openaiResponsesProtocolFrameToSSEFrame, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent, type ClientOpenAIResponsesStreamEvent, type ClientResponseResource } from '@floway-dev/protocols/openai-responses'; import { toInternalDebugError } from '@floway-dev/provider'; @@ -31,17 +32,17 @@ import { toInternalDebugError } from '@floway-dev/provider'; export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, framing: OpenAIResponsesStreamFraming) => defineStage< Fields<'ingress.chat.openaiResponses.wantsStream'>, Fields<'ingress.chat.openaiResponses.wantsStream'>, - Fields<'ingress.chat.openaiResponses.wantsStream' | 'response.chat.openaiResponses' | 'response.http.headers'>, - Fields<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames'>, + Fields<'ingress.chat.openaiResponses.wantsStream' | 'response.chat.openaiResponses' | 'response.http.headers' | 'response.chat.openaiResponses.streamedUsage'>, + Fields<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.openaiResponses.streamedUsage'>, ChatServices >({ name: 'emitOpenAIResponses', through: { request: { needs: ['ingress.chat.openaiResponses.wantsStream'], consumes: [], provides: [] }, response: { - needs: ['response.chat.openaiResponses', 'response.http.headers'], + needs: ['response.chat.openaiResponses', 'response.http.headers', OPENAI_RESPONSES_STREAMED_USAGE], consumes: ['response.chat.openaiResponses', 'response.http.headers'], - provides: ['response.chat.clientFrames', 'response.chat.openaiResponses.rendered', 'response.http.status', 'response.http.headers'], + provides: ['response.chat.clientFrames', 'response.chat.openaiResponses.rendered', 'response.http.status', 'response.http.headers', OPENAI_RESPONSES_STREAMED_USAGE], }, }, execute: async (facts, next, use) => { @@ -77,11 +78,11 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra // Everything this protocol writes back into its own frames, in one place because every // layer of it rewrites them and below the fold there would be nothing left to rewrite: - // the terminal restated from the items that actually closed, the turn's own state sealed - // into the carrier a follow-up comes back on, each complete item stored under its exact - // id beneath one response id this gateway minted, and the resource completed to what the - // schema requires of it. It runs before the fold rather than beside it, so a client that - // did not ask to stream is answered with the object the persisted frames add up to. + // the turn's own state sealed into the carrier a follow-up comes back on, each complete + // item stored under its exact id beneath one response id this gateway minted, and the resource + // completed to what the schema requires of it. It runs before the fold rather than beside it, + // so a client that did not ask to stream is answered with the object the persisted frames + // add up to. const egress = wrapOpenAIResponsesClientEgress( answer.frames as AsyncIterable>, use.gateway, @@ -96,8 +97,8 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra // recorded here rather than where it is framed — one tee for the family, whatever writes // what it hands up. Reading is what records, so a transport that stopped early records // exactly what it took. - const frames = recordStream(egress, use.gateway.dump); if (!back['ingress.chat.openaiResponses.wantsStream']) { + const frames = recordStream(egress, use.gateway.dump); try { return { ...rest, @@ -112,21 +113,26 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra // Nothing has gone out yet, so the fault is still a status. A turn the gateway could // not finish — the snapshot the next turn would read, most often — is not one it can // answer, and the client is told what broke rather than handed the half that arrived. + use.gateway.dump?.failed(error); return { ...rest, 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move(internalErrorEnvelope(error)), 'response.http.status': 502, + [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], Promise.resolve(true))), }; } } + const clientStream = completeStream(egress, framing, error => { use.gateway.dump?.failed(error); }); + const frames = recordStream(clientStream.frames, use.gateway.dump); return { ...rest, 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move(framing === 'sse' ? renderSSE(frames) : frames), 'response.http.status': 200, + [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], clientStream.failed)), }; }, }); @@ -162,41 +168,51 @@ const streamFailedEvent = (announced: ClientResponseResource, error: unknown): C } as ClientOpenAIResponsesStreamEvent; }; -/** Every OpenAI Responses event has an SSE form of its own, so the render is a straight map. What - * it adds is the two endings a client already being streamed to can be given. The ordinary - * one is the terminator: the client's stream ends on the literal `[DONE]` payload whether or - * not the upstream's stream carried one, because that is what the transport reads to know - * the turn is over. +/** The client's terminal frames are protocol content, so they precede recording and transport + * framing. A stream that finishes carries its own terminator, while an interrupted one carries + * the failure under the response id already announced to that client. * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx?plain=1#L84 - * - * The other is a break — an upstream that died mid-stream, a turn that could not be stored, - * a stream that ran out before saying how it ended. The status went out with the headers, so - * the failure has to be said in the protocol's own words, and it is said *instead of* the - * terminator: a stream that ended on `[DONE]` is a stream that finished. */ -const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ - // The frames the client reads are a reframing of the ones the record holds, so this key - // points at that same stream rather than at nothing. - ...streamReferenceOf(frames), - [Symbol.asyncIterator]: () => (async function* () { - // The last resource the client was shown, which is the one a `response.failed` restates: - // the id it names has to be the id the client already saw this turn under. + */ +const completeStream = ( + frames: AsyncIterable>, + framing: OpenAIResponsesStreamFraming, + reportFailure: (error: unknown) => void, +): { frames: AsyncIterable>; failed: Promise } => { + let settle!: (failed: boolean) => void; + const failed = new Promise(resolve => { settle = resolve; }); + const stream = (async function* () { let announced: ClientResponseResource | undefined; + let failed = false; try { for await (const frame of frames) { - // The upstream's own terminator is not the client's — the ending stops reading at the - // turn's terminal event, and one terminator is written below however the frames ended. if (frame.type === 'done') continue; if ('response' in frame.event) announced = frame.event.response; - yield openaiResponsesProtocolFrameToSSEFrame(frame); + yield frame; } - yield openaiResponsesProtocolFrameToSSEFrame(doneFrame()); + if (framing === 'sse') yield doneFrame(); } catch (error) { - yield sseFrame(JSON.stringify(streamErrorEvent(error)), 'error'); - // Nothing was announced when the break came before the first resource-bearing event, - // and there is no response to restate as failed. - if (announced !== undefined) { - yield openaiResponsesProtocolFrameToSSEFrame(eventFrame(streamFailedEvent(announced, error))); - } + failed = true; + reportFailure(error); + if (framing === 'events') throw error; + yield eventFrame(streamErrorEvent(error)); + if (announced !== undefined) yield eventFrame(streamFailedEvent(announced, error)); + } finally { + settle(failed); } + })(); + return { frames: { [Symbol.asyncIterator]: () => stream }, failed }; +}; + +// Persistence and client projection can fail after the upstream terminal has settled its +// usage. Settlement waits for both boundaries and retains the measured upstream quantities. +const withClientVerdict = (reading: Deferred | null, failed: Promise): Deferred | null => + reading === null ? null : defer(Promise.all([reading, failed]).then(([outcome, failed]) => ({ ...outcome, failed: outcome.failed || failed }))); + +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ + // The frames the client reads are a reframing of the ones the record holds, so this key + // points at that same stream rather than at nothing. + ...streamReferenceOf(frames), + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) yield openaiResponsesProtocolFrameToSSEFrame(frame); })(), }); From aeb9c93140ce79ac30af55250335795d0d3ef410 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 03:19:28 +0800 Subject: [PATCH 114/156] refactor(search): carry hosted subrequest content through facts Run dispatch planning and backend execution inside the independent search pipeline. Commands, filters, settings and conversation enter its immutable record, while provider handles and page caches remain runtime services. Preserve eager dispatch before searching frames and close failed child records while propagating the original error. Verify the recorded content and successful/error lifecycles, with all 314 hosted search regressions passing. --- .../hosted-tools/search-sub-request_test.ts | 52 +++++ .../hosted-tools/search-sub-request.ts | 29 +-- .../search-sub-request/execute.ts | 76 +++++++- .../hosted-tools/search-sub-request/facts.ts | 17 +- .../search-sub-request/pipeline.ts | 2 +- .../search-sub-request/services.ts | 11 +- .../hosted-tools/web-search.ts | 179 +++--------------- 7 files changed, 186 insertions(+), 180 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts new file mode 100644 index 0000000000..069478425e --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts @@ -0,0 +1,52 @@ +import { expect, test } from 'vitest'; + +import type { WebSearchRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts'; +import { runWebSearchSubRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts'; +import { initDumpBroker, initDumpStore } from '../../../../../src/dump/registry.ts'; +import { openRunDump } from '../../../../../src/dump/run-sink.ts'; +import type { ApiKey } from '../../../../../src/repo/types.ts'; +import { eventsOf, installDumpStubs } from '../../../../dump/test-fixtures.ts'; +import { flushBackground, trackBackground } from '../../../../test-utils/background-tracker.ts'; +import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; + +const apiKey: ApiKey = { id: 'search-key', userId: 1, name: 'Search key', key: 'floway-search-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; +const request: WebSearchRequest = { + commands: { search_query: [{ q: 'pipeline streams' }] }, toolName: 'web_search', iterationCount: 1, + filters: { allowedDomains: ['example.com'], maxResults: 20 }, includeSearchActionSources: true, + settings: { search_context_size: 'medium' }, input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'Find the documentation' }] }], +}; + +for (const failed of [false, true]) { + test(`web search records backend content before dispatch and closes its ${failed ? 'failed' : 'successful'} child run`, async () => { + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); + const fault = new Error('search transport failed', { cause: new Error('socket reset') }); + let dispatched = false; + const invocation = runWebSearchSubRequest(parent, request, { + session: { pageCache: new Map(), getProvider: () => { throw new Error('local provider must not dispatch in alpha mode'); }, apiKeyId: apiKey.id }, + executeAlpha: async (content, action) => { + dispatched = true; + expect(content).toEqual(request); + expect(Object.isFrozen(content.commands)).toBe(true); + expect(action).toEqual({ type: 'search', query: 'pipeline streams', queries: ['pipeline streams'] }); + if (failed) throw fault; + return { action, results: [], outputText: 'documentation' }; + }, + }); + if (failed) await expect(invocation).rejects.toBe(fault); + else expect(await invocation).toMatchObject({ outputText: 'documentation' }); + expect(dispatched).toBe(true); + await flushBackground(); + expect(dumps.stored).toHaveLength(1); + const record = dumps.stored[0]!.record; + expect(record.meta.path).toBe('/alpha/search'); + expect(record.meta.status).toBe(failed ? 502 : 200); + const read = createRunReader(); + const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); + const entered = decoded.find(event => event?.facts && 'request.webSearch.canonical' in event.facts); + expect(entered?.facts?.['request.webSearch.canonical']).toEqual(request); + expect(new TextDecoder().decode(record.events)).not.toContain('searchCall'); + }); +} diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts index f2e1aefd3e..2b14afcc88 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts @@ -1,5 +1,6 @@ +import type { WebSearchRequest } from './search-sub-request/facts.ts'; import { webSearchSubRequestPipeline } from './search-sub-request/pipeline.ts'; -import type { SearchCall } from './search-sub-request/services.ts'; +import type { WebSearchRuntime } from './search-sub-request/services.ts'; import { prologueFor } from '../../../pipeline/serve.ts'; import type { GatewayCtx, AttemptState } from '../../../shared/gateway-ctx.ts'; import type { WebSearchCallIR } from '../../../tools/web-search/operations.ts'; @@ -14,7 +15,8 @@ import { run, move } from '@floway-dev/pipeline'; */ export const runWebSearchSubRequest = async ( parent: GatewayCtx, - call: SearchCall, + request: WebSearchRequest, + webSearch: WebSearchRuntime, ): Promise => { const attempt: AttemptState = { timing: { firstOutputTokenAt: null, upstreamCallStartedAt: null }, telemetry: undefined }; const dump = parent.dump?.openSubRequest({ method: 'POST', path: '/alpha/search' }, false, attempt.timing) ?? null; @@ -26,13 +28,18 @@ export const runWebSearchSubRequest = async ( }; const prologue = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }, dump); - const { facts, drain } = await run( - webSearchSubRequestPipeline, - move({ 'request.webSearch.action': 'search' }) as never, - { ...prologue.services, searchCall: call } as never, - ); - // Nothing streams out of a search, so the run is over the moment it answers. - await drain(); - dump?.finalize(200, 0); - return facts['response.webSearch.ir']; + try { + const { facts, drain } = await run( + webSearchSubRequestPipeline, + move({ 'request.webSearch.canonical': request }), + { ...prologue.services, webSearch }, + ); + await drain(); + dump?.finalize(200, 0); + return facts['response.webSearch.ir']; + } catch (error) { + dump?.failed(error); + dump?.finalize(502, 0); + throw error; + } }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts index 06ca308f76..036ba28146 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts @@ -1,9 +1,13 @@ -import type { Fields } from './facts.ts'; -import type { SearchServices } from './services.ts'; +import type { Fields, WebSearchRequest } from './facts.ts'; +import type { SearchServices, WebSearchRuntime } from './services.ts'; +import { assertLocalWebSearchSupport, executeOperationToIr, parseWebSearchOperations, runBackendSearchMulti, schemaErrorIr, startBatchFetch, UnsupportedLocalWebSearchFeatureError, unsupportedLocalWebSearchFeatureIr, type WebSearchCallIR, type WebSearchOperation } from '../../../../tools/web-search/operations.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesWebSearchAction } from '@floway-dev/protocols/openai-responses'; + +const ITERATION_CAP = 30; export const runWebSearchCall = defineStage< - Fields<'request.webSearch.action'>, + Fields<'request.webSearch.canonical'>, Fields<'response.webSearch.ir'> & { 'response.usage.billable': readonly never[] }, SearchServices >({ @@ -11,9 +15,67 @@ export const runWebSearchCall = defineStage< return: { provides: ['response.webSearch.ir', 'response.usage.billable'] }, execute: async (facts, use) => move({ ...facts, - 'response.webSearch.ir': await use.searchCall(), - // No model was called, so there is no entity to bill: what the backend charges is accounted - // per api key by the operations as they run. + 'response.webSearch.ir': await execute(facts['request.webSearch.canonical'], use.webSearch), + // Search billing belongs to the backend operations; no model entity is called here. 'response.usage.billable': [], - }) as never, + }), }); + +const execute = async (request: WebSearchRequest, runtime: WebSearchRuntime): Promise => { + const { commands, toolName, iterationCount } = request; + const parsed = parseWebSearchOperations(commands); + if (iterationCount > ITERATION_CAP) { + return schemaErrorIr( + 'tool budget exhausted', + 'Tool call budget exhausted', + `Web search iteration limit (${ITERATION_CAP}) reached. Further web_search calls in this response will return this same error. Summarize what you have already learned, and continue the task using other available tools (shell, file inspection, prior knowledge) or directly answer based on what you've gathered.`, + ); + } + if (parsed.kind === 'malformed' || parsed.ops.length === 0) { + return schemaErrorIr( + 'malformed dispatcher call arguments', + 'Malformed arguments', + 'Error: arguments must be a JSON object with sub-property arrays (search_query[], open[], find[]).', + ); + } + if (runtime.executeAlpha !== undefined) { + const first = parsed.ops[0]; + let action: OpenAIResponsesWebSearchAction; + if (first.kind === 'search') { + const queries = parsed.ops.filter((op): op is Extract => op.kind === 'search').map(op => op.query); + action = queries.length === 1 + ? { type: 'search', query: queries[0], queries } + : { type: 'search', query: queries.join(' | '), queries }; + } else if (first.kind === 'open') { + action = { type: 'open_page', url: first.url }; + } else if (first.kind === 'find') { + action = { type: 'find_in_page', url: first.url, pattern: first.pattern }; + } else { + action = { type: 'search', query: Object.keys(commands).join(', ') }; + } + return await runtime.executeAlpha(request, action); + } + try { + assertLocalWebSearchSupport(commands); + } catch (error) { + if (error instanceof UnsupportedLocalWebSearchFeatureError) { + return unsupportedLocalWebSearchFeatureIr({ type: 'search', query: Object.keys(commands).join(', ') }, error.message); + } + throw error; + } + const session = { ...runtime.session, filters: request.filters, includeSearchActionSources: request.includeSearchActionSources }; + if (parsed.ops.length > 1 && parsed.ops.every(op => op.kind === 'search' && op.error === undefined)) { + return await runBackendSearchMulti(parsed.ops as Extract[], session); + } + if (parsed.ops.length > 1) { + return schemaErrorIr( + 'ambiguous dispatcher call', + 'Ambiguous tool call', + `Error: ambiguous \`${toolName}\` tool call — each function_call maps to one web_search_call. ` + + 'Multiple `search_query` entries are fine (they collapse into one search). ' + + 'For `open`/`find`, or any mix of kinds, split into one call per `open[]` entry, `find[]` entry, or `search_query[]` batch.', + ); + } + const batch = await startBatchFetch(parsed, session); + return await executeOperationToIr(parsed.ops[0], session, batch); +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts index 34b416335d..d2a1b6d3cc 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts @@ -1,10 +1,19 @@ import type { GatewayFacts } from '../../../../pipeline/facts.ts'; -import type { WebSearchCallIR } from '../../../../tools/web-search/operations.ts'; +import type { WebSearchCallIR, WebSearchFilters } from '../../../../tools/web-search/operations.ts'; +import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; + +export interface WebSearchRequest { + commands: Record; + toolName: string; + iterationCount: number; + filters: WebSearchFilters; + includeSearchActionSources: boolean; + settings: Record; + input: OpenAIResponsesInputItem[]; +} -/** What one search call is, and what it came to. */ export interface WebSearchSubRequestFacts extends GatewayFacts { - 'request.webSearch.action': 'search'; - /** What the backend answered, in the shape the hosted-tool item is built from. */ + 'request.webSearch.canonical': WebSearchRequest; 'response.webSearch.ir': WebSearchCallIR; } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/pipeline.ts index d7dd464c16..3d52cf07b4 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/pipeline.ts @@ -3,7 +3,7 @@ import type { Fields } from './facts.ts'; import { writeSettlement } from '../../../../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; -export const webSearchSubRequestPipeline: Pipeline, Fields<'response.webSearch.ir'>> = +export const webSearchSubRequestPipeline: Pipeline, Fields<'response.webSearch.ir'>> = compose('webSearchSubRequest', [ writeSettlement(() => false), runWebSearchCall, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts index 2a5c7d387b..db095e1b1c 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts @@ -1,6 +1,11 @@ +import type { WebSearchRequest } from './facts.ts'; import type { GatewayServices } from '../../../../pipeline/services.ts'; -import type { WebSearchCallIR } from '../../../../tools/web-search/operations.ts'; +import type { WebSearchCallIR, WebSearchExecutionSession } from '../../../../tools/web-search/operations.ts'; +import type { OpenAIResponsesWebSearchAction } from '@floway-dev/protocols/openai-responses'; -export type SearchCall = () => Promise; +export interface WebSearchRuntime { + readonly session: Omit; + readonly executeAlpha?: (request: WebSearchRequest, action: OpenAIResponsesWebSearchAction) => Promise; +} -export interface SearchServices extends GatewayServices { readonly searchCall: SearchCall } +export interface SearchServices extends GatewayServices { readonly webSearch: WebSearchRuntime } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts index 843e246235..d508e3e3fd 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts @@ -1,5 +1,6 @@ +import type { WebSearchRuntime } from './search-sub-request/services.ts'; import { runWebSearchSubRequest } from './search-sub-request.ts'; -import { type HostedToolLoopState, type HostedToolOutputItem, type HostedToolRegistration } from './types.ts'; +import { type HostedToolOutputItem, type HostedToolRegistration } from './types.ts'; import { shortId } from '../../../../shared/short-id.ts'; import { truncatePreservingCodePoints } from '../../../shared/text.ts'; import { executeAlphaSearch } from '../../../tools/web-search/alpha-search/execution.ts'; @@ -8,24 +9,13 @@ import { loadWebSearchConfig } from '../../../tools/web-search/config.ts'; import { normalizeDomainEntry } from '../../../tools/web-search/domain-normalize.ts'; import { actionSearchQueries, - assertLocalWebSearchSupport, CONTEXT_SIZE_TO_MAX_RESULTS, DEFAULT_SEARCH_CONTEXT_SIZE, - executeOperationToIr, isSearchContextSize, maxResultsForContextSize, - parseWebSearchOperations, renderWebSearchCallOutput, - runBackendSearchMulti, - schemaErrorIr, - startBatchFetch, - UnsupportedLocalWebSearchFeatureError, - unsupportedLocalWebSearchFeatureIr, - type ParsedWebSearchOperations, type WebSearchCallIR, - type WebSearchExecutionSession, type WebSearchFilters, - type WebSearchOperation, } from '../../../tools/web-search/operations.ts'; import { resolveConfiguredWebSearchProvider } from '../../../tools/web-search/provider.ts'; import type { ConfiguredWebSearchProvider } from '../../../tools/web-search/types.ts'; @@ -578,134 +568,6 @@ export const transformInputItemsForWebSearch = ( return out; }; -// The dispatcher's execution session plus the one wire-shaping flag that lives -// only on the OpenAI Responses side. -interface HostedToolState extends WebSearchExecutionSession { - // Set when the client passed `include: ["web_search_call.results"]` on - // the request. Native OpenAI Responses gates the `results` field on this - // include token; the dispatcher follows suit on the wire item — but the IR - // (and therefore `payload.private`) always carries the real results - // so a subsequent turn echoing the item id can be hydrated regardless. - includeSearchResults: boolean; - executeAlpha?: (commands: Record, action: OpenAIResponsesWebSearchAction) => Promise; -} - -const ITERATION_CAP = 30; - -const eagerResolver = (promise: Promise): (() => Promise) => { - const settled = promise.then( - value => ({ ok: true as const, value }), - error => ({ ok: false as const, error }), - ); - return async () => { - const result = await settled; - if (result.ok) return result.value; - throw result.error; - }; -}; - -const planDispatchSlots = ( - parsed: ParsedWebSearchOperations, - commands: Record, - toolName: string, - state: HostedToolState, - loopState: HostedToolLoopState, -): { id: string; resolve: () => Promise } => { - if (loopState.iterationCount > ITERATION_CAP) { - return { - id: synthesizeWebSearchCallId(), - resolve: async () => schemaErrorIr( - 'tool budget exhausted', - 'Tool call budget exhausted', - `Web search iteration limit (${ITERATION_CAP}) reached. Further web_search calls in this response will return this same error. Summarize what you have already learned, and continue the task using other available tools (shell, file inspection, prior knowledge) or directly answer based on what you've gathered.`, - ), - }; - } - - if (parsed.kind === 'malformed' || parsed.ops.length === 0) { - return { - id: synthesizeWebSearchCallId(), - resolve: async () => schemaErrorIr( - 'malformed dispatcher call arguments', - 'Malformed arguments', - 'Error: arguments must be a JSON object with sub-property arrays (search_query[], open[], find[]).', - ), - }; - } - - const executeAlpha = state.executeAlpha; - if (executeAlpha !== undefined) { - const first = parsed.ops[0]; - let action: OpenAIResponsesWebSearchAction; - if (first.kind === 'search') { - const queries = parsed.ops.filter((op): op is Extract => op.kind === 'search').map(op => op.query); - action = queries.length === 1 - ? { type: 'search', query: queries[0], queries } - : { type: 'search', query: queries.join(' | '), queries }; - } else if (first.kind === 'open') { - action = { type: 'open_page', url: first.url }; - } else if (first.kind === 'find') { - action = { type: 'find_in_page', url: first.url, pattern: first.pattern }; - } else { - action = { type: 'search', query: Object.keys(commands).join(', ') }; - } - return { id: synthesizeWebSearchCallId(), resolve: eagerResolver(executeAlpha(commands, action)) }; - } - - try { - assertLocalWebSearchSupport(commands); - } catch (error) { - if (error instanceof UnsupportedLocalWebSearchFeatureError) { - return { - id: synthesizeWebSearchCallId(), - resolve: async () => unsupportedLocalWebSearchFeatureIr( - { type: 'search', query: Object.keys(commands).join(', ') }, - error.message, - ), - }; - } - return { - id: synthesizeWebSearchCallId(), - resolve: async () => { throw error; }, - }; - } - - // Multi-`search_query` entries collapse into one wsc with a multi-query - // action (`{type:'search', queries:[...]}`) — protocol-native and the - // only same-kind shape that fits in one wsc. Require every entry to - // parse cleanly; one malformed entry forces the model to fix all of - // them rather than silently dropping a search. - if (parsed.ops.length > 1 && parsed.ops.every(op => op.kind === 'search' && op.error === undefined)) { - const searchOps = parsed.ops as Array>; - return { - id: synthesizeWebSearchCallId(), - resolve: eagerResolver(runBackendSearchMulti(searchOps, state)), - }; - } - - // Any other multi-op shape cannot reduce to a single wsc action: `open` - // and `find` actions each carry one url/pattern, and mixed kinds have - // incompatible action types. Surface as ambiguous and let the model - // split into independent calls. - if (parsed.ops.length > 1) { - return { - id: synthesizeWebSearchCallId(), - resolve: async () => schemaErrorIr( - 'ambiguous dispatcher call', - 'Ambiguous tool call', - `Error: ambiguous \`${toolName}\` tool call — each function_call maps to one web_search_call. ` - + 'Multiple `search_query` entries are fine (they collapse into one search). ' - + 'For `open`/`find`, or any mix of kinds, split into one call per `open[]` entry, `find[]` entry, or `search_query[]` batch.', - ), - }; - } - - return { - id: synthesizeWebSearchCallId(), - resolve: eagerResolver(startBatchFetch(parsed, state).then(async batch => await executeOperationToIr(parsed.ops[0], state, batch))), - }; -}; - export const webSearchHostedTool: HostedToolRegistration = async (invocation, gatewayCtx) => { if (invocation.targetApi === 'openaiResponses' && !providerModelOf(invocation.candidate).enabledFlags.has('openai-responses-web-search-shim')) { return { type: 'inactive' }; @@ -732,18 +594,16 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga let configuredProvider: Promise | undefined; const hosted = tools.filter(isHostedWebSearchTool).at(-1); const settings = alphaSearchSettingsFromHosted(hosted); - const state: HostedToolState = { - filters, + const session: WebSearchRuntime['session'] = { pageCache: new Map(), getProvider: () => { configuredProvider ??= Promise.resolve(resolveConfiguredWebSearchProvider(webSearchConfig)); return configuredProvider; }, apiKeyId: gatewayCtx.apiKeyId, - includeSearchResults: includeArray.includes('web_search_call.results'), - includeSearchActionSources: includeArray.includes('web_search_call.action.sources'), ...(gatewayCtx.abortSignal !== undefined ? { signal: gatewayCtx.abortSignal } : {}), }; + let executeAlpha: WebSearchRuntime['executeAlpha']; if (webSearchConfig.passthroughOpenAiSearch.enabled) { const dispatcher = resolveAlphaSearchDispatcher({ config: webSearchConfig.passthroughOpenAiSearch, @@ -752,16 +612,17 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga runtimeLocation: gatewayCtx.runtimeLocation, }); const sessionId = crypto.randomUUID(); - state.executeAlpha = async (commands, action) => await executeAlphaSearch({ + executeAlpha = async (request, action) => await executeAlphaSearch({ dispatcher: await dispatcher, sessionId, - commands, - settings, - input: invocation.payload.input, + commands: request.commands, + settings: request.settings, + input: request.input, action, signal: gatewayCtx.abortSignal, }); } + const webSearch: WebSearchRuntime = { session, ...(executeAlpha === undefined ? {} : { executeAlpha }) }; return { type: 'active', @@ -775,7 +636,15 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga buildFunctionTool, dispatcher: ({ intercepted, loopState }) => { const commands = intercepted.arguments ?? {}; - const slot = planDispatchSlots(parseWebSearchOperations(intercepted.arguments), commands, intercepted.name, state, loopState); + const id = synthesizeWebSearchCallId(); + const search = runWebSearchSubRequest(gatewayCtx, { + commands, toolName: intercepted.name, iterationCount: loopState.iterationCount, filters, + includeSearchActionSources: includeArray.includes('web_search_call.action.sources'), + settings, input: invocation.payload.input, + }, webSearch).then( + value => ({ ok: true as const, value }), + error => ({ ok: false as const, error }), + ); const functionCallItem: OpenAIResponsesFunctionToolCallItem = { type: 'function_call', call_id: intercepted.callId, @@ -788,22 +657,24 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga status: 'completed', }; return [{ - id: slot.id, + id, startItem: { type: 'web_search_call', status: 'in_progress' }, startEvents: [ { type: 'response.web_search_call.in_progress' }, { type: 'response.web_search_call.searching' }, ], // Its own run: its own prologue, its own settlement, its own record. The - // planning above stays with the turn — what a call resolves to is the turn's - // decision — and what runs the backend is a run of its own. + // commands, filters, and conversation enter the child record before its backend + // executes; live caches and provider handles remain in its services. run: async function* run() { - const ir = await runWebSearchSubRequest(gatewayCtx, slot.resolve); + const result = await search; + if (!result.ok) throw result.error; + const ir = result.value; // `results` is gated on the client's `include` // opt-in to match native OpenAI Responses' default wire // shape; the IR keeps them either way for the // private-payload round-trip. - const item: HostedToolOutputItem & Omit = state.includeSearchResults + const item: HostedToolOutputItem & Omit = includeArray.includes('web_search_call.results') ? { type: 'web_search_call', status: 'completed', action: ir.action, results: ir.results } : { type: 'web_search_call', status: 'completed', action: ir.action }; const privatePayload: WebSearchCallPrivatePayload = { From 117a0a182701ccf8830634ec7145288b20ded86e Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 03:42:03 +0800 Subject: [PATCH 115/156] refactor(images): run hosted calls through shared model stages Carry prompts, configuration and portable image sources through preparation facts. Share Images resolution, failover, protocol metering and settlement with hosted generation and edit calls; only their response projection differs. Preserve rate-limit retries as a stage that retains every actual call observation for one settlement. Close active hosted lifecycles when their parent returns and release unread image bodies before awaiting their usage. Cover generation/edit content recording, consumed and unread replies, cancellation and retry billing. All 384 focused image and hosted-tool tests, gateway typecheck and lint pass. --- .../image-generation-integration_test.ts | 40 +- .../hosted-tools/image-generation_test.ts | 23 +- .../hosted-tools/image-sub-request_test.ts | 85 ++- .../hosted-tools/shared_test.ts | 30 + .../hosted-tools/image-generation.ts | 627 +----------------- .../hosted-tools/image-sub-request.ts | 62 +- .../hosted-tools/image-sub-request/dial.ts | 66 -- .../hosted-tools/image-sub-request/emit.ts | 94 +++ .../hosted-tools/image-sub-request/facts.ts | 16 +- .../image-sub-request/pipeline.ts | 26 +- .../hosted-tools/image-sub-request/prepare.ts | 38 ++ .../hosted-tools/image-sub-request/request.ts | 186 ++++++ .../hosted-tools/image-sub-request/result.ts | 133 ++++ .../hosted-tools/image-sub-request/retry.ts | 78 +++ .../image-sub-request/services.ts | 12 - .../openai-responses/hosted-tools/shared.ts | 20 +- .../src/data-plane/openai-images/pipeline.ts | 30 +- .../src/data-plane/openai-images/target.ts | 2 +- 18 files changed, 739 insertions(+), 829 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/shared_test.ts delete mode 100644 packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/dial.ts create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts delete mode 100644 packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/services.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts index f5e5958d50..e7fca7d226 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts @@ -26,7 +26,7 @@ interface BackendStub { nextEdits: Response[]; // When set, the next `enumerateModelCandidates` call returns this // shape verbatim instead of the default single in-test candidate; lets - // a test drive `resolveImageCandidate`'s failure branches. + // a test drive shared resolution failure branches. nextResolutionOverride: { candidates: readonly unknown[]; sawModel: boolean; failedUpstreams: readonly string[] } | null; } @@ -43,6 +43,7 @@ const defaultCandidates = vi.hoisted(() => () => [{ upstreamId: 'u', kind: 'custom', name: 'mock-image', + inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, @@ -192,10 +193,7 @@ const drain = async (result: ExecuteResult { repo = new InMemoryRepo(); - // resolveImageCandidate still calls createPerRequestFetcher to satisfy the - // production code path. Seed the in-memory repo with the mocked candidate's - // upstream id so the fetcher mapper resolves it instead of throwing - // "unknown upstream id: u". + // Attribution resolves the candidate's upstream metadata while writing the child run. await saveUpstreamForTest(repo.upstreams, { id: 'u', kind: 'custom', @@ -494,25 +492,27 @@ test('image transcoding failure becomes a terminal image tool failure', async () assertStringIncludes(item.error.message, 'codec down'); }); -test('retries on 429 and surfaces the eventual success', async () => { - // Two 429s then success; the orchestrator should see one completed image_generation_call. +for (const stream of [false, true]) test(`retries on 429 and settles every actual call before its ${stream ? 'streamed' : 'single-body'} success`, async () => { + const usage = { input_tokens: 9, output_tokens: 4 }; stub.nextGenerations = [ - rateLimitResponse(1), - rateLimitResponse(1), - jsonResponse('T0s='), + rateLimitResponse(1), rateLimitResponse(1), + stream ? sseResponse([JSON.stringify({ type: 'image_generation.completed', b64_json: 'T0s=', usage })]) : jsonResponse('T0s=', usage), ]; - const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ - callTurn(0, 'call_1', 'a cat'), - messageTurn('done'), + const pending: Promise[] = []; + const ctx = mockChatGatewayCtx({ wantsStream: true, backgroundScheduler: work => { pending.push(work); } }); + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }], 'auto', stream ? { partial_images: 1 } : {}), ctx, scriptedRun([ + callTurn(0, 'call_1', 'a cat'), messageTurn('done'), ])); const events = await drain(result); - + await Promise.all(pending); assertEquals(stub.generationsCalls.length, 3); const igcDone = events.find(e => e.type === 'response.output_item.done' && (e as { item: { type: string } }).item.type === 'image_generation_call'); assert(igcDone !== undefined); const item = (igcDone as { item: { status: string; result: string } }).item; assertEquals(item.status, 'completed'); assertEquals(item.result, 'T0s='); + const calls = (await repo.usage.listAll()).filter(row => row.model === 'gpt-image-2'); + assertEquals(calls.reduce((total, row) => total + row.requests, 0), 3); }); test('gives up after MAX_RATE_LIMIT_RETRIES on persistent 429 and surfaces a failed item', async () => { @@ -558,12 +558,12 @@ test('does not retry non-rate-limit upstream failures', async () => { assertEquals(item.error.code, 'invalid_value'); }); -// resolveImageCandidate failure branches: each variant is normalized into +// Shared image resolution failure branches: each variant is normalized into // a terminal `image_generation_call` item with `status: 'failed'` and a // specific error.code so the orchestrator can distinguish unknown-model // from existing-but-unsupported. -test('resolveImageCandidate renders model_not_found when no upstream knows the model id', async () => { +test('shared image resolution renders model_not_found when no upstream knows the model id', async () => { stub.nextResolutionOverride = { candidates: [], sawModel: false, failedUpstreams: [] }; const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), @@ -578,10 +578,10 @@ test('resolveImageCandidate renders model_not_found when no upstream knows the m const item = (igcDone as { item: { status: string; error: { code: string; message: string } } }).item; assertEquals(item.status, 'failed'); assertEquals(item.error.code, 'model_not_found'); - assert(item.error.message.includes("No upstream provides model 'gpt-image-2'"), `unexpected message: ${item.error.message}`); + assert(item.error.message.includes('Model gpt-image-2 is not available on any configured upstream.'), `unexpected message: ${item.error.message}`); }); -test('resolveImageCandidate renders model_not_supported when sawModel=true but no candidate is an image kind', async () => { +test('shared image resolution renders model_not_supported when sawModel=true but no candidate is an image kind', async () => { // Mirrors the resolver's "id exists in some catalog but the kind filter // dropped it" signal — sawModel=true, candidates=[]. stub.nextResolutionOverride = { candidates: [], sawModel: true, failedUpstreams: [] }; @@ -597,10 +597,10 @@ test('resolveImageCandidate renders model_not_supported when sawModel=true but n const item = (igcDone as { item: { status: string; error: { code: string; message: string } } }).item; assertEquals(item.status, 'failed'); assertEquals(item.error.code, 'model_not_supported'); - assert(item.error.message.includes("Model 'gpt-image-2' is not an image model."), `unexpected message: ${item.error.message}`); + assert(item.error.message.includes('Model gpt-image-2 does not support the /images/generations endpoint.'), `unexpected message: ${item.error.message}`); }); -test('resolveImageCandidate renders model_not_supported when image-kind candidates exist but none expose the openaiImagesGenerations endpoint', async () => { +test('shared image resolution renders model_not_supported when image-kind candidates exist but none expose the openaiImagesGenerations endpoint', async () => { // The resolver produced an image-kind candidate but its `endpoints` does // not include the per-endpoint key the request needs (openaiImagesGenerations). stub.nextResolutionOverride = { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts index ed3aa186b1..40fed5f22f 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts @@ -6,18 +6,16 @@ import { inspectImageSources, DEFAULT_IMAGE_MODEL, type ImageGenerationConfig, - type ImageOutcome, imageGenerationHostedTool, - imageTerminal, isHostedImageGenerationTool, - parseImageStreamEvent, - parseRetryAfterMs, prepareImageGenerationConfig, resolveImageOperation, FUNCTION_TOOL_NAME, synthesizeImageGenerationCallId, transformInputItemsForImageGeneration, } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts'; +import { imageTerminal, projectImageStreamEvent, type ImageOutcome } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts'; +import { parseRetryAfterMs } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts'; import { initRepo } from '../../../../../src/repo/index.ts'; import { InMemoryRepo } from '../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; @@ -483,33 +481,30 @@ test('imageTerminal on failure emits a failed item and no closing events', () => assertEquals(endEvents.length, 0); }); -// ── parseImageStreamEvent ── - -test('parseImageStreamEvent maps generations and edits partial/completed/error with backend echo', () => { - const genPartial = parseImageStreamEvent(JSON.stringify({ type: 'image_generation.partial_image', partial_image_index: 1, b64_json: PNG_B64, background: 'opaque', output_format: 'png', quality: 'low', size: '1024x1024' })); +test('native image events project generations and edits partial/completed/error with backend echo', () => { + const genPartial = projectImageStreamEvent({ type: 'image_generation.partial_image', partial_image_index: 1, b64_json: PNG_B64, background: 'opaque', output_format: 'png', quality: 'low', size: '1024x1024' }); assert(genPartial?.kind === 'partial'); assertEquals(genPartial.index, 1); assertEquals(genPartial.b64, PNG_B64); assertEquals(genPartial.echo, { background: 'opaque', output_format: 'png', quality: 'low', size: '1024x1024' }); - const editPartial = parseImageStreamEvent(JSON.stringify({ type: 'image_edit.partial_image', partial_image_index: 0, b64_json: PNG_B64 })); + const editPartial = projectImageStreamEvent({ type: 'image_edit.partial_image', partial_image_index: 0, b64_json: PNG_B64 }); assert(editPartial?.kind === 'partial'); assertEquals(editPartial.echo, {}); - const completed = parseImageStreamEvent(JSON.stringify({ type: 'image_generation.completed', b64_json: PNG_B64, usage: { total_tokens: 1 }, quality: 'high' })); + const completed = projectImageStreamEvent({ type: 'image_generation.completed', b64_json: PNG_B64, usage: { total_tokens: 1 }, quality: 'high' }); assert(completed?.kind === 'completed'); assertEquals(completed.b64, PNG_B64); assertEquals(completed.echo.quality, 'high'); - const err = parseImageStreamEvent(JSON.stringify({ type: 'error', error: { type: 'image_generation_server_error', code: 'image_generation_failed', message: 'boom' } })); + const err = projectImageStreamEvent({ type: 'error', error: { type: 'image_generation_server_error', code: 'image_generation_failed', message: 'boom' } }); assert(err?.kind === 'error'); assertEquals(err.error.code, 'image_generation_failed'); assertEquals(err.error.retryable, true); }); -test('parseImageStreamEvent returns null for non-JSON or unrelated events', () => { - assertEquals(parseImageStreamEvent('[DONE]'), null); - assertEquals(parseImageStreamEvent(JSON.stringify({ type: 'image_generation.queued' })), null); +test('native unrelated image events do not become hosted lifecycle events', () => { + assertEquals(projectImageStreamEvent({ type: 'image_generation.queued' }), null); }); test('prepareImageGenerationConfig rejects a present-but-invalid model', () => { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts index 9736645189..72c936d173 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts @@ -1,40 +1,69 @@ -import { expect, test } from 'vitest'; +import { afterEach, expect, test, vi } from 'vitest'; +import type { ImageGenerationRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts'; import { runImageGenerationSubRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts'; -import type { HostedToolTerminal } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/types.ts'; +import * as resolution from '../../../../../src/data-plane/providers/resolution.ts'; import { initDumpBroker, initDumpStore } from '../../../../../src/dump/registry.ts'; import { openRunDump } from '../../../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../../../src/repo/index.ts'; import type { ApiKey } from '../../../../../src/repo/types.ts'; import { eventsOf, installDumpStubs } from '../../../../dump/test-fixtures.ts'; +import { InMemoryRepo } from '../../../../repo/memory.ts'; import { flushBackground, trackBackground } from '../../../../test-utils/background-tracker.ts'; import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; +import { stubModelCandidate } from '@floway-dev/test-utils'; const apiKey: ApiKey = { id: 'image-key', userId: 1, name: 'Image key', key: 'floway-image-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; +afterEach(() => { vi.restoreAllMocks(); }); -test('a recorded image lifecycle preserves its terminal result and independent run', async () => { - const dumps = installDumpStubs(initDumpStore, initDumpBroker); - const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); - const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); - const terminal: HostedToolTerminal = { item: { type: 'image_generation_call', id: 'image', status: 'completed', result: 'pixels' }, endEvents: [] }; - const progress = { type: 'response.image_generation_call.in_progress', item_id: 'image' }; - const run = await runImageGenerationSubRequest(parent, 'generate', settle => (async function* () { - yield progress; - settle([], false, undefined); - return terminal; - })()); - expect(await run.lifecycle.next()).toEqual({ done: false, value: progress }); - const result = await run.lifecycle.next(); - expect(result.done).toBe(true); - expect(result.value).toBe(terminal); - await run.drain(); - await flushBackground(); - expect(dumps.stored).toHaveLength(1); - const record = dumps.stored[0]!.record; - expect(record.meta.path).toBe('/images/generations'); - const read = createRunReader(); - const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); - expect(decoded.flatMap(event => event?.frames ?? [])).toEqual([{ type: 'event', event: progress }, { type: 'event', event: terminal }]); - expect(new TextDecoder().decode(record.events)).toContain('request.imageGeneration.action'); - expect(new TextDecoder().decode(record.events)).not.toContain('request.imageGeneration.call'); -}); +for (const action of ['generate', 'edit'] as const) for (const consume of [false, true]) { + test(`a recorded hosted image ${action} carries source facts through the full model chain when ${consume ? 'consumed' : 'returned unread'}`, async () => { + const repo = new InMemoryRepo(); + initRepo(repo); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); + const candidate = stubModelCandidate({ model: { id: 'image-model', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } } }); + const prefix = action === 'edit' ? 'image_edit' : 'image_generation'; + const response = () => new Response([ + { type: `${prefix}.partial_image`, partial_image_index: 0, b64_json: 'preview' }, + { type: `${prefix}.completed`, b64_json: 'pixels', usage: { input_tokens: 7, output_tokens: 3 } }, + ].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } }); + const generate = vi.spyOn(candidate.provider.instance, 'callOpenAIImagesGenerations').mockImplementation(async () => ({ response: response(), modelKey: 'backend-image' })); + const edit = vi.spyOn(candidate.provider.instance, 'callOpenAIImagesEdits').mockImplementation(async () => ({ response: response(), modelKey: 'backend-image' })); + vi.spyOn(resolution, 'enumerateModelCandidates').mockResolvedValue({ candidates: [candidate], sawModel: true, failedUpstreams: [] }); + const request: ImageGenerationRequest = { + prompt: 'draw a tree', action, config: { model: 'image-model', action, quality: 'high', partial_images: 1 }, + sources: action === 'edit' ? [{ bytes: new Uint8Array([1, 2, 3]), mimeType: 'image/png' }] : [], + }; + const run = await runImageGenerationSubRequest(parent, request); + if (consume) { + expect(await run.lifecycle.next()).toMatchObject({ done: false, value: { type: 'response.image_generation_call.partial_image', partial_image_index: 0, partial_image_b64: 'preview' } }); + expect(await run.lifecycle.next()).toMatchObject({ done: true, value: { item: { type: 'image_generation_call', status: 'completed', action, revised_prompt: 'draw a tree', result: 'pixels' }, endEvents: [{ type: 'response.image_generation_call.completed' }] } }); + } else { + await run.lifecycle.return(undefined as never); + } + await run.drain(); + await flushBackground(); + expect(generate).toHaveBeenCalledTimes(action === 'generate' ? 1 : 0); + expect(edit).toHaveBeenCalledTimes(action === 'edit' ? 1 : 0); + expect(parent.attempt.telemetry).toBeUndefined(); + expect(dumps.stored).toHaveLength(1); + const record = dumps.stored[0]!.record; + expect(record.meta.path).toBe(action === 'edit' ? '/images/edits' : '/images/generations'); + const read = createRunReader(); + const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); + expect(decoded.find(event => event?.facts && 'request.imageGeneration.canonical' in event.facts)?.facts?.['request.imageGeneration.canonical']).toEqual({ ...request, sources: action === 'edit' ? [{ bytes: { bytes: 'AQID' }, mimeType: 'image/png' }] : [] }); + const prepared = decoded.find(event => event?.facts && 'request.openaiImages.canonical' in event.facts)?.facts?.['request.openaiImages.canonical']; + expect(prepared).toMatchObject({ operation: action === 'edit' ? 'edits' : 'generations', parameters: { prompt: 'draw a tree', n: 1, quality: 'high', stream: true, partial_images: 1 } }); + if (action === 'edit') expect(prepared).toMatchObject({ images: [{ kind: 'file', file: { fileName: 'image_0.png', mediaType: 'image/png', bytes: { bytes: 'AQID' } } }] }); + const entered = eventsOf(record).filter(event => event.type === 'stage.entered').map(event => event.name); + expect(entered).toEqual(['emitHostedImageGeneration', 'prepareHostedImageGeneration', 'writeSettlement', 'resolveCandidates', 'failover', 'retryRateLimitedImages', 'callOpenAIImagesUpstream']); + const frames = decoded.flatMap(event => event?.frames ?? []); + if (consume) expect(frames).toContainEqual({ type: 'event', event: { item: { type: 'image_generation_call', status: 'completed', action, result: 'pixels', revised_prompt: 'draw a tree' }, endEvents: [{ type: 'response.image_generation_call.completed' }] } }); + else expect(frames).toEqual([]); + const observations = await repo.usage.listAll(); + expect(observations).toMatchObject([{ model: 'image-model', modelKey: 'backend-image', requests: 1 }]); + }); +} diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/shared_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/shared_test.ts new file mode 100644 index 0000000000..29f7395f1c --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/shared_test.ts @@ -0,0 +1,30 @@ +import { expect, test } from 'vitest'; + +import { createMergeState, materializeHostedToolItems } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/shared.ts'; +import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/types.ts'; +import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; + +test('returning hosted materialization closes the active child lifecycle after its delivered partial frame', async () => { + let closed = false; + const frames = materializeHostedToolItems([{ + slots: [{ + intercepted: { callId: 'call', name: 'image_generation', arguments: { prompt: 'tree' } }, + outputIndex: 0, + slot: { + id: 'image', startItem: { type: 'image_generation_call', status: 'in_progress' }, startEvents: [], + run: () => (async function* (): AsyncGenerator { + try { + yield { type: 'response.image_generation_call.partial_image', partial_image_index: 0, partial_image_b64: 'preview' }; + throw new Error('a cancelled lifecycle must not request another event'); + } finally { + closed = true; + } + })(), + }, + }], + }], createMergeState(), mockChatGatewayCtx().store); + expect(await frames.next()).toMatchObject({ done: false, value: { type: 'event', event: { type: 'response.image_generation_call.partial_image', partial_image_b64: 'preview', item_id: 'image', output_index: 0 } } }); + expect(closed).toBe(false); + await frames.return(); + expect(closed).toBe(true); +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts index 9942abd480..5418086470 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts @@ -1,17 +1,11 @@ -import { type SettleImageCall } from './image-sub-request/services.ts'; +import { portableImageRequest, type ImageBackendConfig } from './image-sub-request/request.ts'; +import { imageTerminal, isRetryableImageError } from './image-sub-request/result.ts'; import { runImageGenerationSubRequest } from './image-sub-request.ts'; -import type { HostedToolLifecycleEvent, HostedToolOutputItem, HostedToolRegistration, HostedToolTerminal } from './types.ts'; -import { sleep } from '../../../../shared/sleep.ts'; -import type { BillableEntity } from '../../../pipeline/facts.ts'; -import { enumerateModelCandidates } from '../../../providers/resolution.ts'; -import { stampUpstreamCallStart } from '../../../shared/attempt-timing.ts'; -import { appendFailedUpstreams } from '../../../shared/failed-upstreams.ts'; -import type { AttemptState, GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { PerformanceTelemetryContext } from '../../../shared/telemetry/performance.ts'; -import { tokenUsageFromOpenAIImagesBody, tokenUsageMeasurement } from '../../../shared/telemetry/usage.ts'; +import type { HostedToolLifecycleEvent, HostedToolRegistration, HostedToolTerminal } from './types.ts'; +import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../shared/external-image-loader.ts'; -import { dimensionsFromBytes, getImageProcessor, type BackgroundScheduler } from '@floway-dev/platform'; -import { decodeForgivingBase64, encodeHex, isImageMediaType, mediaTypeEssence, parseSSEStream } from '@floway-dev/protocols/common'; +import { dimensionsFromBytes } from '@floway-dev/platform'; +import { decodeForgivingBase64, isImageMediaType, mediaTypeEssence } from '@floway-dev/protocols/common'; import { collectOpenAIResponsesToolEntries, createRandomOpenAIResponsesItemId, @@ -22,10 +16,9 @@ import { type OpenAIResponsesInputImage, type OpenAIResponsesInputImageGenerationCall, type OpenAIResponsesInputItem, - type OpenAIResponsesOutputImageGenerationCall, type OpenAIResponsesTool, } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf, type Fetcher, type OpenAIImagesEditsRequest, type Provider, type ModelCandidate, type ProviderModel } from '@floway-dev/provider'; +import { providerModelOf } from '@floway-dev/provider'; export const FUNCTION_TOOL_NAME = 'image_generation'; @@ -54,30 +47,6 @@ const ALLOWED_MODERATIONS = new Set(['auto', 'low']); const ALLOWED_ACTIONS = new Set(['generate', 'edit', 'auto']); const ALLOWED_INPUT_FIDELITY = new Set(['high', 'low']); -// gpt-image-* `/images/edits` accepts only these input image mimetypes; a live -// Azure probe confirmed png/jpeg/webp succeed while gif is rejected with -// `unsupported_file_mimetype`. Native OpenAI Responses accepts the same GIF and -// re-encodes it before editing, so the dispatcher mirrors that behavior through the -// platform image processor. Common aliases are folded onto the backend form. -type EditMime = 'image/png' | 'image/jpeg' | 'image/webp'; - -const EDIT_MIME_ALIASES: Record = { - 'image/jpg': 'image/jpeg', - 'image/pjpeg': 'image/jpeg', - 'image/x-png': 'image/png', -}; -// The canonical edit-supported mimetype for a source, or null when the -// standalone endpoint requires local WebP transcoding first. -const editSupportedMime = (mime: string): EditMime | null => { - const canonical = EDIT_MIME_ALIASES[mime] ?? mime; - return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' - ? canonical - : null; -}; - -const editFileExt = (mime: EditMime): string => - mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; - // The public `image_generation` tool-config surface. Azure rejects any other // field with `unknown_parameter`, so the dispatcher mirrors that strictness rather // than silently forwarding unknown fields (which would diverge from the @@ -101,17 +70,11 @@ export const isHostedImageGenerationTool = (tool: OpenAIResponsesTool): tool is export const canonicalizeImageGenerationTool = (raw: OpenAIResponsesTool): OpenAIResponsesHostedTool | undefined => isHostedImageGenerationTool(raw) ? raw : undefined; -// A base64-data-URL or bare-base64 image source bound for an edit call. -// Bytes are held in a concrete ArrayBuffer so they can be wrapped in a Blob. interface ImageSource { bytes: ArrayBuffer; mimeType: string; } -interface PreparedImageSource extends ImageSource { - mimeType: EditMime; -} - interface RemoteImageSource { wireUrl: string; invalidUrlParam: string; @@ -123,39 +86,6 @@ type ImageSourceReference = ImageSource | RemoteImageSource; const isRemoteImageSource = (source: ImageSourceReference): source is RemoteImageSource => 'wireUrl' in source; -const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { - const keyBySource = new Map>(); - const preparedByContent = new Map>(); - return await Promise.all(sources.map(async source => { - const mimeType = editSupportedMime(source.mimeType); - if (mimeType !== null) return { bytes: source.bytes, mimeType }; - - let keyPromise = keyBySource.get(source); - if (keyPromise === undefined) { - keyPromise = crypto.subtle.digest('SHA-256', source.bytes).then(buffer => { - const digest = encodeHex(new Uint8Array(buffer)); - return `${source.mimeType}\u0000${digest}`; - }); - keyBySource.set(source, keyPromise); - } - const key = await keyPromise; - - let prepared = preparedByContent.get(key); - if (prepared === undefined) { - // Native OpenAI Responses accepts formats such as GIF through its multimodal - // preprocessing, while the standalone edits endpoint accepts only - // PNG/JPEG/WebP. Re-encode locally to preserve the hosted-tool behavior. - // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 - prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { - const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; - return { bytes, mimeType: 'image/webp' } satisfies PreparedImageSource; - }); - preparedByContent.set(key, prepared); - } - return await prepared; - })); -}; - const base64ToArrayBuffer = (b64: string): ArrayBuffer => { const bytes = decodeForgivingBase64(b64); return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer; @@ -245,53 +175,12 @@ const decodeInputImageDataUrl = ( // The orchestrator-visible tool config the dispatcher layers onto the backend // call. Mirrors Azure: the orchestrator only chooses `prompt`; everything // here is read from the client's hosted-tool entry and applied by the dispatcher. -export interface ImageGenerationConfig { - model: string; - size?: string; - quality?: string; - output_format?: 'png' | 'jpeg'; - background?: 'transparent' | 'opaque' | 'auto'; - moderation?: 'auto' | 'low'; - output_compression?: number; - // When > 0, the backend call is issued with `stream:true` and each - // progressively-rendered preview the backend emits is relayed as a native - // `image_generation_call.partial_image` frame. When 0/absent the backend - // is called non-streaming and no preview frames are produced. - partial_images?: number; - input_fidelity?: 'high' | 'low'; - // Inpainting mask materialized once at validation, forwarded to - // /images/edits as the standalone `mask` part. `file_id` masks are not - // supported (rejected at validation) — resolving them needs the Files API. +export interface ImageGenerationConfig extends ImageBackendConfig { mask?: ImageSourceReference; - action: 'generate' | 'edit' | 'auto'; } type MaterializedImageGenerationConfig = Omit & { mask?: ImageSource }; -const prepareEditRequest = async ( - sources: readonly ImageSource[], - config: MaterializedImageGenerationConfig, -): Promise<{ sources: readonly PreparedImageSource[]; mask?: PreparedImageSource }> => { - const originals = [...sources]; - if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); - const prepared = await prepareEditSources(originals); - const bySource = new Map(); - for (const [index, source] of originals.entries()) { - const wireSource = prepared[index]; - if (wireSource === undefined) throw new Error('Missing prepared image edit source'); - bySource.set(source, wireSource); - } - const wireSources = sources.map(source => { - const wireSource = bySource.get(source); - if (wireSource === undefined) throw new Error('Missing prepared image edit source'); - return wireSource; - }); - if (config.mask === undefined) return { sources: wireSources }; - const mask = bySource.get(config.mask); - if (mask === undefined) throw new Error('Missing prepared image edit mask'); - return { sources: wireSources, mask }; -}; - interface PrepareConfigError { message: string; param: string; @@ -840,501 +729,12 @@ export const resolveImageOperation = ( // continue. The full upstream error shape (type/code/message) is preserved so // the orchestrator can distinguish transient overload from a terminal // content-policy block. -type ImageError = { type: string; code: string; message: string; retryable: boolean }; - -// Server-resolved tool config echoed by the backend on both the partial_image -// frames and the final result (`background:"auto"` becomes the concrete value -// the server picked, etc.). Read straight off the backend rather than inferred -// from the request, so what we surface matches what was actually rendered. -interface EchoFields { - output_format?: 'png' | 'jpeg'; - quality?: 'low' | 'medium' | 'high'; - background?: 'transparent' | 'opaque'; - size?: string; -} - -export type ImageOutcome = - | { ok: true; b64: string; echo: EchoFields } - | { ok: false; error: ImageError }; - -// Project the server-resolved echo fields out of a backend payload (a response -// JSON body or an SSE event). Each field is validated against the public enum -// so a surprising backend value is dropped rather than echoed verbatim. -const extractEcho = (source: unknown): EchoFields => { - if (source === null || typeof source !== 'object') return {}; - const s = source as Record; - const echo: EchoFields = {}; - if (s.output_format === 'png' || s.output_format === 'jpeg') echo.output_format = s.output_format; - if (s.quality === 'low' || s.quality === 'medium' || s.quality === 'high') echo.quality = s.quality; - if (s.background === 'transparent' || s.background === 'opaque') echo.background = s.background; - if (typeof s.size === 'string') echo.size = s.size; - return echo; -}; - -const RETRYABLE_IMAGE_ERROR_CODES = new Set([ - 'EngineOverloaded', 'server_error', 'image_generation_server_error', 'image_generation_failed', -]); - -const isRetryableImageError = (code: string, type?: string): boolean => - RETRYABLE_IMAGE_ERROR_CODES.has(code) || (type !== undefined && RETRYABLE_IMAGE_ERROR_CODES.has(type)); - -const errorFromBody = (body: string, status: number): { type?: string; code: string; message: string } => { - try { - const parsed = JSON.parse(body) as { error?: { message?: unknown; code?: unknown; type?: unknown } }; - const err = parsed.error; - if (err !== undefined && err !== null) { - return { - ...(typeof err.type === 'string' ? { type: err.type } : {}), - message: typeof err.message === 'string' ? err.message : `Image backend returned HTTP ${status}`, - code: typeof err.code === 'string' ? err.code : `upstream_${status}`, - }; - } - } catch (e) { - if (!(e instanceof SyntaxError)) throw e; - } - return { message: `Image backend returned HTTP ${status}`, code: `upstream_${status}` }; -}; - -// Per-request inputs the dispatcher's backend call needs. Captured in the -// registration closure so the dispatcher reads none of the turn it was -// registered for. Edit sources are NOT captured here — they are re-collected -// from the payload the current descent is sending, so an image generated in an -// earlier turn (fed back as an `input_image`) becomes editable in a later one. `imageDispatchCount` bounds how many real backend image -// calls one response may issue. interface HostedToolState { config: MaterializedImageGenerationConfig; - apiKeyId: string; - upstreamIds: readonly string[] | null; - backgroundScheduler: BackgroundScheduler; - runtimeLocation: string; - downstreamAbortSignal: AbortSignal | undefined; - imageDispatchCount: number; - /** The turn's own context, which each call opens its own run over. */ gateway: GatewayCtx; + imageDispatchCount: number; } -/** What one backend answer is billable for, in the shape settlement writes a row from. A body - * that reported nothing bills nothing, which is a different statement from reporting zero. */ -const billedImage = (provider: Provider, model: ProviderModel, modelKey: string, responseBody: unknown): readonly BillableEntity[] => { - const usage = tokenUsageFromOpenAIImagesBody(responseBody); - if (usage === null) return []; - const measurement = tokenUsageMeasurement(usage); - return [{ - identity: { model: model.id, upstream: provider.upstreamId, modelKey, pricing: model.pricing ?? null }, - quantities: measurement.quantities, - pricingFacts: measurement.pricingFacts, - }]; -}; - -const buildGenerationsBody = (prompt: string, config: ImageGenerationConfig, stream: boolean): Record => ({ - prompt, - // Public OpenAI Responses tool config forbids `n`, but the private standalone - // backend call always requests a single image, mirroring Azure's - // single-image OpenAI Responses behavior. - n: 1, - // `response_format` is intentionally not sent: gpt-image-* always returns - // base64 (`data[0].b64_json`) and rejects `response_format`, so the inline - // extraction below reads `b64_json` directly. - ...(config.size !== undefined ? { size: config.size } : {}), - ...(config.quality !== undefined ? { quality: config.quality } : {}), - ...(config.output_format !== undefined ? { output_format: config.output_format } : {}), - ...(config.background !== undefined ? { background: config.background } : {}), - ...(config.moderation !== undefined ? { moderation: config.moderation } : {}), - ...(config.output_compression !== undefined ? { output_compression: config.output_compression } : {}), - ...(stream ? { stream: true, partial_images: config.partial_images } : {}), -}); - -const buildEditsRequest = ( - prompt: string, - config: ImageGenerationConfig, - sources: readonly PreparedImageSource[], - mask: PreparedImageSource | undefined, - stream: boolean, -): OpenAIImagesEditsRequest => { - const parameters: Record = { - prompt, - n: 1, - ...(config.size === undefined ? {} : { size: config.size }), - ...(config.quality === undefined ? {} : { quality: config.quality }), - ...(config.output_format === undefined ? {} : { output_format: config.output_format }), - ...(config.background === undefined ? {} : { background: config.background }), - ...(config.moderation === undefined ? {} : { moderation: config.moderation }), - ...(config.output_compression === undefined ? {} : { output_compression: config.output_compression }), - ...(config.input_fidelity === undefined ? {} : { input_fidelity: config.input_fidelity }), - ...(stream ? { stream: true, partial_images: config.partial_images } : {}), - }; - const images = sources.map((source, index) => ({ - type: 'upload' as const, - file: new File([source.bytes], `image_${index}.${editFileExt(source.mimeType)}`, { type: source.mimeType }), - })); - const maskFile = mask === undefined - ? undefined - : new File([mask.bytes], `mask.${editFileExt(mask.mimeType)}`, { type: mask.mimeType }); - return { - images, - ...(maskFile === undefined ? {} : { mask: { type: 'upload' as const, file: maskFile } }), - parameters, - }; -}; - -const serverError = (e: unknown): ImageError => ({ - type: 'image_generation_error', - message: e instanceof Error ? e.message : String(e), - code: 'server_error', - retryable: true, -}); - -// Resolve the candidate that serves the configured image model for the -// target endpoint. A resolution/availability failure is normalized into -// an `ImageError` so the caller always produces a terminal image item. -const resolveImageCandidate = async ( - isEdit: boolean, - state: HostedToolState, -): Promise<{ ok: true; candidate: ModelCandidate } | { ok: false; error: ImageError }> => { - const endpointKey = isEdit ? 'openaiImagesEdits' : 'openaiImagesGenerations'; - const endpointPath = isEdit ? '/images/edits' : '/images/generations'; - let resolution; - try { - resolution = await enumerateModelCandidates({ - upstreamIds: state.upstreamIds, - model: state.config.model, - kind: 'image', - scheduler: state.backgroundScheduler, - runtimeLocation: state.runtimeLocation, - }); - } catch (e) { - return { ok: false, error: serverError(e) }; - } - const match = resolution.candidates.find(c => c.model.endpoints[endpointKey] !== undefined); - if (match !== undefined) { - return { ok: true, candidate: match }; - } - // Split on the resolver's `sawModel` signal the same way serve-prep.ts - // does for chat: an unknown model id ("model_not_found", 404-shaped) vs - // a model that exists under some catalog but cannot serve this op - // ("model_not_supported"). The latter splits further on whether the - // resolver's kind filter rejected the id (sawModel=true, candidates=[]: - // id exists but not as an image model) or the per-endpoint key did - // (candidates non-empty: image-kind upstreams exist but none expose the - // requested edits/generations endpoint). - if (!resolution.sawModel) { - return { - ok: false, - error: { - type: 'image_generation_error', - message: appendFailedUpstreams(`No upstream provides model '${state.config.model}'.`, resolution.failedUpstreams), - code: 'model_not_found', - retryable: false, - }, - }; - } - const message = resolution.candidates.length === 0 - ? `Model '${state.config.model}' is not an image model.` - : `No upstream supporting the ${endpointPath} endpoint provides model '${state.config.model}'.`; - return { - ok: false, - error: { - type: 'image_generation_error', - message: appendFailedUpstreams(message, resolution.failedUpstreams), - code: 'model_not_supported', - retryable: false, - }, - }; -}; - -// 60s cap matches the per-minute refill window of Azure TPM/RPM and -// openai.com tier image quotas — same clamp openai-python applies in -// [`_calculate_retry_timeout`](https://github.com/openai/openai-python/blob/d76d8c11c1da9f97aa8a0aaee8ccd44d2bc8f5e7/src/openai/_base_client.py#L789). -const RETRY_CAP_MS = 60_000; -const MAX_RATE_LIMIT_RETRIES = 2; - -// Header priority matches openai-python's `_parse_retry_after_header` with -// Azure's `x-ms-retry-after-ms` alias added. Treats <= 0 as "no hint" so the -// gpt-image-1 `retry-after: 0.0` quirk falls back to backoff instead of -// pretending the quota is free. -export const parseRetryAfterMs = (headers: Headers): number | null => { - for (const name of ['retry-after-ms', 'x-ms-retry-after-ms']) { - const raw = headers.get(name); - if (raw === null) continue; - const ms = Number(raw); - if (Number.isFinite(ms) && ms > 0) return ms; - } - const ra = headers.get('retry-after'); - if (ra !== null) { - const seconds = Number(ra); - if (Number.isFinite(seconds) && seconds > 0) return seconds * 1000; - const httpDateMs = Date.parse(ra); - if (!Number.isNaN(httpDateMs)) { - const delta = httpDateMs - Date.now(); - if (delta > 0) return delta; - } - } - return null; -}; - -// On 429, sleep for the upstream's retry hint (or jittered exponential -// backoff when absent) and replay the same backend call up to -// MAX_RATE_LIMIT_RETRIES times. The returned `response` always has a fresh, -// unread body — intermediate failed responses are drained inside the loop so -// the underlying socket can be reused while we sleep. -const issueImageCall = async ( - provider: Provider, - model: ProviderModel, - fetcher: Fetcher, - prompt: string, - editRequest: OpenAIImagesEditsRequest | null, - config: ImageGenerationConfig, - state: HostedToolState, - stream: boolean, - attempt: AttemptState, -): Promise<{ response: Response; modelKey: string }> => { - for (let retry = 0; ; retry++) { - const opts = { - fetcher, - waitUntil: state.backgroundScheduler, - headers: new Headers(), - // Stamp this image sub-call's OWN perf slot — never ctx.attempt — - // so the outer OpenAI Responses turn's upstream-call stamp is preserved. - // Perf recording lives at the sub-call's terminal boundary in - // streamImageGeneration; the retry loop overwrites this slot each - // retry so it reflects the dispatch that actually returned. - wrapUpstreamCall: stampUpstreamCallStart(attempt.timing), - }; - const { response, modelKey } = await (editRequest === null - ? provider.instance.callOpenAIImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts) - : provider.instance.callOpenAIImagesEdits(model, editRequest, state.downstreamAbortSignal, opts)); - if (response.status !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) return { response, modelKey }; - - // 25% jitter desynchronizes parallel callers so a burst of orchestrator - // turns doesn't all re-issue at the same instant. - const base = 1000 * 2 ** retry; - const backoffMs = base + Math.random() * base * 0.25; - const delayMs = Math.min(parseRetryAfterMs(response.headers) ?? backoffMs, RETRY_CAP_MS); - await response.text().catch(() => undefined); - await sleep(delayMs, state.downstreamAbortSignal); - } -}; - -// Consume a non-streaming backend response (partial_images = 0) into an -// outcome. Transport/backend failures become `{ok:false}` rather than -// throwing, so the caller always produces a terminal image item. -const consumeImageResponse = async ( - provider: Provider, - model: ProviderModel, - modelKey: string, - response: Response, - state: HostedToolState, - billed: BillableEntity[], -): Promise => { - const text = await response.text(); - if (!response.ok) { - const { type, code, message } = errorFromBody(text, response.status); - return { ok: false, error: { type: type ?? 'image_generation_error', code, message, retryable: isRetryableImageError(code, type) } }; - } - let parsed: unknown; - try { - parsed = JSON.parse(text); - } catch { - return { ok: false, error: { type: 'image_generation_error', message: 'Image backend returned a non-JSON success body.', code: 'server_error', retryable: true } }; - } - const b64 = (() => { - if (parsed === null || typeof parsed !== 'object') return null; - const data = (parsed as { data?: unknown }).data; - if (!Array.isArray(data) || data.length === 0) return null; - const first = data[0] as { b64_json?: unknown }; - return typeof first.b64_json === 'string' ? first.b64_json : null; - })(); - if (b64 === null) { - return { ok: false, error: { type: 'image_generation_error', message: 'Image backend response did not contain image bytes.', code: 'server_error', retryable: true } }; - } - billed.push(...billedImage(provider, model, modelKey, parsed)); - return { ok: true, b64, echo: extractEcho(parsed) }; -}; - -// Build the completed/failed `image_generation_call` output item plus its -// closing events. On success the final bytes ride `output_item.done.item.result` -// and a `response.image_generation_call.completed` closes the item; on failure -// neither `.completed` nor any `.partial_image` is emitted — only the failed -// `output_item.done`. -// -// `revised_prompt` is set to the orchestrator's prompt: the standalone images -// backend does no prompt rewriting and returns no `revised_prompt`, and the -// orchestrator's emitted prompt IS already its refined rewrite (it plays the -// role Azure's native flow gives the orchestrator), so it is the faithful -// source for this field. -export const imageTerminal = ( - prompt: string, - action: 'generate' | 'edit', - outcome: ImageOutcome, -): HostedToolTerminal => { - if (!outcome.ok) { - const item: HostedToolOutputItem & Omit = { - type: 'image_generation_call', - status: 'failed', - revised_prompt: prompt, - error: { message: outcome.error.message, code: outcome.error.code, type: outcome.error.type }, - }; - return { item, endEvents: [] }; - } - - const item: HostedToolOutputItem & Omit = { - type: 'image_generation_call', - status: 'completed', - action, - result: outcome.b64, - revised_prompt: prompt, - ...outcome.echo, - }; - return { item, endEvents: [{ type: 'response.image_generation_call.completed' }] }; -}; - -// One standalone-images SSE data line, folded into a backend-agnostic signal. -// The generations and edits endpoints use distinct event prefixes -// (`image_generation.*` vs `image_edit.*`); only the suffix is matched here. -type ImageStreamSignal = - | { kind: 'partial'; index: number; b64: string; echo: EchoFields } - | { kind: 'completed'; b64: string | undefined; usage: unknown; echo: EchoFields } - | { kind: 'error'; error: ImageError } - | null; - -export const parseImageStreamEvent = (data: string): ImageStreamSignal => { - let evt: { type?: unknown; partial_image_index?: unknown; b64_json?: unknown; usage?: unknown; error?: unknown }; - try { - evt = JSON.parse(data); - } catch { - return null; - } - const type = typeof evt.type === 'string' ? evt.type : ''; - if (type.endsWith('.partial_image')) { - return { - kind: 'partial', - index: typeof evt.partial_image_index === 'number' ? evt.partial_image_index : 0, - b64: typeof evt.b64_json === 'string' ? evt.b64_json : '', - echo: extractEcho(evt), - }; - } - if (type.endsWith('.completed')) { - return { kind: 'completed', b64: typeof evt.b64_json === 'string' ? evt.b64_json : undefined, usage: evt.usage, echo: extractEcho(evt) }; - } - if (type === 'error') { - const err = evt.error as { message?: unknown; code?: unknown; type?: unknown } | undefined; - const code = typeof err?.code === 'string' ? err.code : 'server_error'; - const errType = typeof err?.type === 'string' ? err.type : 'image_generation_error'; - return { - kind: 'error', - error: { type: errType, code, message: typeof err?.message === 'string' ? err.message : 'Image backend stream reported an error.', retryable: isRetryableImageError(code, errType) }, - }; - } - return null; -}; - -// Drive the backend and produce the deferred slot lifecycle: relay each -// progressively-rendered preview as a native `partial_image` frame, then -// return the terminal `image_generation_call` item. partial_images = 0 (or -// absent) takes a single non-streaming round-trip and yields no preview frames. -// -// Every sub-call records its OWN perf row under operation='image_generation' -// or 'image_edit' via a local AttemptState distinct from ctx.attempt (which -// belongs to the outer OpenAI Responses turn). firstOutputTokenAt stays null — image -// backends are single-body from the perf model's point of view, so the row -// lands in the neutral bucket with an honest requests + errors count and no -// synthesized TTFT. Resolution failures record no row: no upstream was ever -// dispatched. -const streamImageGeneration = ( - prompt: string, - action: 'generate' | 'edit', - isEdit: boolean, - sources: readonly ImageSource[], - state: HostedToolState, - settle: SettleImageCall, - billed: BillableEntity[], -) => async function* (): AsyncGenerator { - const resolved = await resolveImageCandidate(isEdit, state); - if (!resolved.ok) { - // No candidate means no upstream was called, so there is nothing billed and no performance - // context to attribute — but the reading still settles, because a run waits for what it - // started and a call that resolved nothing has finished. - settle([], true, undefined); - return imageTerminal(prompt, action, { ok: false, error: resolved.error }); - } - const { provider, fetcher } = resolved.candidate; - const model = providerModelOf(resolved.candidate); - const wantsPartials = (state.config.partial_images ?? 0) > 0; - - const attempt: AttemptState = { timing: { upstreamCallStartedAt: null, firstOutputTokenAt: null }, telemetry: undefined }; - const perfContext: PerformanceTelemetryContext = { - keyId: state.apiKeyId, - model: model.id, - upstream: provider.upstreamId, - operation: isEdit ? 'image_edit' : 'image_generation', - runtimeLocation: state.runtimeLocation, - }; - // The call's own settlement, through the seam every other upstream call settles by: the - // performance sample is attributed to this run's attempt slot rather than the turn's, which is - // what keeps the outer turn's upstream stamp intact. - const finish = (outcome: ImageOutcome): HostedToolTerminal => { - settle(billed, !outcome.ok, perfContext); - return imageTerminal(prompt, action, outcome); - }; - - let response: Response; - let modelKey: string; - try { - let editRequest: OpenAIImagesEditsRequest | null = null; - if (isEdit) { - const prepared = await prepareEditRequest(sources, state.config); - editRequest = buildEditsRequest(prompt, state.config, prepared.sources, prepared.mask, wantsPartials); - } - ({ response, modelKey } = await issueImageCall( - provider, - model, - fetcher, - prompt, - editRequest, - state.config, - state, - wantsPartials, - attempt, - )); - } catch (e) { - return finish({ ok: false, error: serverError(e) }); - } - - if (!wantsPartials) { - return finish(await consumeImageResponse(provider, model, modelKey, response, state, billed)); - } - - if (!response.ok) { - const { type, code, message } = errorFromBody(await response.text(), response.status); - return finish({ ok: false, error: { type: type ?? 'image_generation_error', code, message, retryable: isRetryableImageError(code, type) } }); - } - if (response.body === null) { - return finish({ ok: false, error: { type: 'image_generation_error', message: 'Image backend returned a streaming response with no body.', code: 'server_error', retryable: true } }); - } - - let finalB64: string | undefined; - let finalEcho: EchoFields = {}; - let usage: unknown; - for await (const frame of parseSSEStream(response.body, { signal: state.downstreamAbortSignal })) { - const signal = parseImageStreamEvent(frame.data); - if (signal === null) continue; - if (signal.kind === 'partial') { - yield { type: 'response.image_generation_call.partial_image', partial_image_index: signal.index, partial_image_b64: signal.b64, ...signal.echo }; - } else if (signal.kind === 'completed') { - finalB64 = signal.b64; - finalEcho = signal.echo; - usage = signal.usage; - } else { - return finish({ ok: false, error: signal.error }); - } - } - if (finalB64 === undefined) { - return finish({ ok: false, error: { type: 'image_generation_error', message: 'Image backend stream ended without a completed image.', code: 'server_error', retryable: true } }); - } - billed.push(...billedImage(provider, model, modelKey, { usage })); - return finish({ ok: true, b64: finalB64, echo: finalEcho }); -}; - // Output-as-input round-trip: the multi-turn loop feeds accumulated // `image_generation_call` items back as the next turn's input, and client // histories may echo prior ones. Non-OpenAI-Responses upstreams can't read the item @@ -1497,11 +897,6 @@ export const imageGenerationHostedTool: HostedToolRegistration = async (invocati const state: HostedToolState = { config: materializedConfig, gateway: gatewayCtx, - apiKeyId: gatewayCtx.apiKeyId, - upstreamIds: gatewayCtx.upstreamIds, - backgroundScheduler: gatewayCtx.backgroundScheduler, - runtimeLocation: gatewayCtx.runtimeLocation, - downstreamAbortSignal: gatewayCtx.abortSignal, imageDispatchCount: 0, }; @@ -1562,11 +957,9 @@ export const imageGenerationHostedTool: HostedToolRegistration = async (invocati // Its own run: its own prologue, its own settlement, its own record. What this // yields is that run's lifecycle, spliced into the turn that asked for it. run: () => (async function* (): AsyncGenerator { - const billed: BillableEntity[] = []; const call = await runImageGenerationSubRequest( state.gateway, - operation.action, - (settle: SettleImageCall) => streamImageGeneration(promptArg, operation.action, operation.action === 'edit', sources, state, settle, billed)(), + portableImageRequest(promptArg, operation.action, state.config, sources), ); try { return yield* call.lifecycle; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts index 1310484fb3..829d193973 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts @@ -1,5 +1,5 @@ import { imageGenerationSubRequestPipeline } from './image-sub-request/pipeline.ts'; -import type { ImageCall } from './image-sub-request/services.ts'; +import type { ImageGenerationRequest } from './image-sub-request/request.ts'; import type { HostedToolLifecycleEvent, HostedToolTerminal } from './types.ts'; import { consoleLogSink } from '../../../../runtime/log.ts'; import { prologueFor } from '../../../pipeline/serve.ts'; @@ -16,41 +16,53 @@ import { run, move } from '@floway-dev/pipeline'; */ export const runImageGenerationSubRequest = async ( parent: GatewayCtx, - action: 'generate' | 'edit', - call: ImageCall, + request: ImageGenerationRequest, ): Promise<{ readonly lifecycle: AsyncGenerator; readonly drain: () => Promise; }> => { - const path = action === 'edit' ? '/images/edits' : '/images/generations'; + const path = request.action === 'edit' ? '/images/edits' : '/images/generations'; const attempt: AttemptState = { timing: { firstOutputTokenAt: null, upstreamCallStartedAt: null }, telemetry: undefined }; - const dump = parent.dump?.openSubRequest({ method: 'POST', path }, false, attempt.timing) ?? null; + const wantsStream = (request.config.partial_images ?? 0) > 0; + const dump = parent.dump?.openSubRequest({ method: 'POST', path }, wantsStream, attempt.timing) ?? null; const gateway: GatewayCtx = { ...parent, + wantsStream, requestStartedAt: Date.now(), attempt, dump, }; const prologue = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }, dump); - const { facts, drain } = await run( - imageGenerationSubRequestPipeline, - move({ 'request.imageGeneration.action': action }) as never, - { ...prologue.services, imageCall: call } as never, - ); - return { - lifecycle: facts['response.imageGeneration.lifecycle'], - // The epilogue a served turn gets from the seam, which a sub-request has to be its own: the - // reading resolves when the lifecycle runs out, which is after this run answered, so the row - // is written here rather than in the chain. - drain: async () => { - const reading = facts['response.imageGeneration.streamedUsage']; - if (reading !== null) { - const outcome = await reading; - settleBillable({ ...prologue.services, log: consoleLogSink }, outcome.billable, outcome.failed); - } - await drain(); - dump?.finalize(200, 0); - }, - }; + try { + const { facts, drain } = await run( + imageGenerationSubRequestPipeline(request), + move({ 'request.imageGeneration.canonical': request, 'ingress.http.headers': [] }), + { ...prologue.services }, + ); + return { + lifecycle: facts['response.imageGeneration.lifecycle'], + drain: async () => { + try { + // Releasing the native body also drives an unread image to its usage terminal. + // Waiting for that reading first would prevent the owner from ever starting it. + await drain(); + } catch (error) { + dump?.failed(error); + throw error; + } finally { + const reading = facts['response.openaiImages.streamedUsage']; + if (reading !== null) { + const outcome = await reading; + settleBillable({ ...prologue.services, log: consoleLogSink }, outcome.billable, outcome.failed); + } + dump?.finalize(facts['response.http.status'], 0); + } + }, + }; + } catch (error) { + dump?.failed(error); + dump?.finalize(502, 0); + throw error; + } }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/dial.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/dial.ts deleted file mode 100644 index 01effeb1f8..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/dial.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { RunDump } from '../../../../../dump/run-sink.ts'; -import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../types.ts'; -import type { Fields } from './facts.ts'; -import type { ImageServices } from './services.ts'; -import type { BillableEntity } from '../../../../pipeline/facts.ts'; -import type { StreamOutcome } from '../../../../pipeline/serve.ts'; -import { defineStage, defer, move } from '@floway-dev/pipeline'; -import { eventFrame } from '@floway-dev/protocols/common'; - -const recordLifecycle = (source: AsyncGenerator, dump: RunDump | null): AsyncGenerator => { - if (dump === null) return source; - const recording = dump.openStream(); - const generator = (async function* () { - let completed = false; - try { - for (;;) { - const step = await source.next(); - recording.frame(eventFrame(step.value)); - if (step.done) { - completed = true; - recording.end(); - return step.value; - } - yield step.value; - } - } finally { - if (!completed) await source.return(undefined as never); - } - })(); - return Object.assign(generator, recording.fact); -}; - -/** - * The ending. It hands up the lifecycle the caller drives, and the reading that lifecycle - * settles — the same shape every streaming family hands up, for the same reason: the numbers - * arrive with the last event. - */ -export const dialImageGeneration = defineStage< - Fields<'request.imageGeneration.action'>, - Fields<'response.imageGeneration.lifecycle' | 'response.imageGeneration.streamedUsage'> & { 'response.usage.billable': readonly BillableEntity[] }, - ImageServices ->({ - name: 'dialImageGeneration', - return: { - provides: ['response.imageGeneration.lifecycle', 'response.imageGeneration.streamedUsage', 'response.usage.billable'], - }, - execute: async (facts, use) => { - let settle!: (outcome: StreamOutcome) => void; - // Declared as this run's own unfinished work, so the runner waits for it at teardown where - // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); - return move({ - ...facts, - 'response.imageGeneration.lifecycle': recordLifecycle(use.imageCall((billable, failed, telemetry) => { - // The sample is attributed to this run's own attempt slot, which is what keeps the - // turn that asked for the image from having its upstream stamp overwritten. - use.gateway.attempt.telemetry = telemetry; - settle({ billable, failed }); - }), use.gateway.dump), - 'response.imageGeneration.streamedUsage': outcome, - // Nothing has been reported when this hands up; what the call turns out to have cost - // arrives through the reading above. - 'response.usage.billable': [], - }) as never; - }, -}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts new file mode 100644 index 0000000000..6e3411178f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts @@ -0,0 +1,94 @@ +import type { Fields } from './facts.ts'; +import { errorFromBody, extractEcho, imageTerminal, isRetryableImageError, projectImageStreamEvent, serverError, type ImageOutcome } from './result.ts'; +import type { RunDump } from '../../../../../dump/run-sink.ts'; +import { isFailure } from '../../../../pipeline/facts.ts'; +import type { GatewayServices } from '../../../../pipeline/services.ts'; +import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../types.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { eventFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIImagesResponse } from '@floway-dev/protocols/openai-images'; + +export const emitHostedImageGeneration = defineStage< + Fields<'request.imageGeneration.canonical'>, + Fields<'request.imageGeneration.canonical'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status'>, + Fields<'response.imageGeneration.lifecycle' | 'response.openaiImages.streamedUsage' | 'response.http.status'>, + GatewayServices +>({ + name: 'emitHostedImageGeneration', + through: { + request: { needs: ['request.imageGeneration.canonical'], consumes: [], provides: [] }, + response: { + needs: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status'], + consumes: ['response.openaiImages.canonical'], provides: ['response.imageGeneration.lifecycle'], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.openaiImages.canonical': answer, ...rest } = back; + const request = facts['request.imageGeneration.canonical']; + const lifecycle = (async function* (): AsyncGenerator { + let outcome: ImageOutcome; + if (isFailure(answer)) { + const error = answer.body === undefined + ? { type: 'image_generation_error', code: answer.status === 404 ? 'model_not_found' : answer.status === 400 ? 'model_not_supported' : 'server_error', message: answer.message } + : errorFromBody(JSON.stringify(answer.body), answer.status); + outcome = { ok: false, error: { ...error, type: error.type ?? 'image_generation_error', retryable: isRetryableImageError(error.code, error.type) } }; + } else if (Symbol.asyncIterator in answer) { + try { + for await (const event of answer) { + const signal = projectImageStreamEvent(event); + if (signal === null) continue; + if (signal.kind === 'partial') { + yield { type: 'response.image_generation_call.partial_image', partial_image_index: signal.index, partial_image_b64: signal.b64, ...signal.echo }; + } else if (signal.kind === 'completed') { + return imageTerminal(request.prompt, request.action, signal.b64 === undefined + ? { ok: false, error: serverError(new Error('Image backend stream ended without a completed image.')) } + : { ok: true, b64: signal.b64, echo: signal.echo }); + } else { + return imageTerminal(request.prompt, request.action, { ok: false, error: signal.error }); + } + } + outcome = { ok: false, error: serverError(new Error('Image backend stream ended without a completed image.')) }; + } catch (error) { + use.gateway.dump?.failed(error); + outcome = { ok: false, error: serverError(error) }; + } + } else { + outcome = imageOutcome(answer); + } + return imageTerminal(request.prompt, request.action, outcome); + })(); + return { ...rest, 'response.imageGeneration.lifecycle': move(recordLifecycle(lifecycle, use.gateway.dump)) }; + }, +}); + +const imageOutcome = (answer: CanonicalOpenAIImagesResponse): ImageOutcome => { + const b64 = answer.images[0]?.base64; + return b64 === undefined + ? { ok: false, error: serverError(new Error('Image backend response did not contain image bytes.')) } + : { ok: true, b64, echo: extractEcho(answer.raw) }; +}; + +const recordLifecycle = (source: AsyncGenerator, dump: RunDump | null): AsyncGenerator => { + if (dump === null) return source; + const recording = dump.openStream(); + const generator = (async function* () { + let completed = false; + try { + for (;;) { + const step = await source.next(); + recording.frame(eventFrame(step.value)); + if (step.done) { + completed = true; + recording.end(); + return step.value; + } + yield step.value; + } + } finally { + if (!completed) await source.return(undefined as never); + } + })(); + return Object.assign(generator, recording.fact); +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts index 1995a1203c..6ca0482b0c 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts @@ -1,16 +1,12 @@ -import type { GatewayFacts } from '../../../../pipeline/facts.ts'; -import type { StreamOutcome } from '../../../../pipeline/serve.ts'; +import type { ImageGenerationRequest } from './request.ts'; +import type { OpenAIImagesFacts } from '../../../../openai-images/facts.ts'; import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../types.ts'; -import type { Deferred } from '@floway-dev/pipeline'; -/** What one image call is, and what it comes to. */ -export interface ImageSubRequestFacts extends GatewayFacts { - 'request.imageGeneration.action': 'generate' | 'edit'; - /** The lifecycle the caller splices into its own answer. */ +export interface ImageSubRequestFacts extends OpenAIImagesFacts { + 'request.imageGeneration.canonical': ImageGenerationRequest; 'response.imageGeneration.lifecycle': AsyncGenerator; - /** What the call turned out to cost, once its events have run out — which is after this run - * has answered, because the caller is what drives them. */ - 'response.imageGeneration.streamedUsage': Deferred | null; } export type Fields = { [P in K]: ImageSubRequestFacts[P] }; +export type ImageSubRequestEntry = Fields<'request.imageGeneration.canonical' | 'ingress.http.headers'>; +export type ImageSubRequestExit = Fields<'response.imageGeneration.lifecycle' | 'response.openaiImages.streamedUsage' | 'response.http.status'>; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/pipeline.ts index 10288733b1..05526e115c 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/pipeline.ts @@ -1,15 +1,15 @@ -import { dialImageGeneration } from './dial.ts'; -import type { Fields } from './facts.ts'; -import { writeSettlement } from '../../../../pipeline/settlement.ts'; +import { emitHostedImageGeneration } from './emit.ts'; +import type { ImageSubRequestEntry, ImageSubRequestExit } from './facts.ts'; +import { prepareHostedImageGeneration } from './prepare.ts'; +import type { ImageGenerationRequest } from './request.ts'; +import { retryRateLimitedImages } from './retry.ts'; +import { callOpenAIImagesUpstream } from '../../../../openai-images/call-upstream.ts'; +import { openaiImagesModelStages } from '../../../../openai-images/pipeline.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; -export const imageGenerationSubRequestPipeline: Pipeline< - Fields<'request.imageGeneration.action'>, - Fields<'response.imageGeneration.lifecycle' | 'response.imageGeneration.streamedUsage'> -> = compose('imageGenerationSubRequest', [ - writeSettlement( - () => false, - handedUp => (handedUp as { 'response.imageGeneration.streamedUsage'?: unknown })['response.imageGeneration.streamedUsage'] !== null, - ), - dialImageGeneration, -]); +export const imageGenerationSubRequestPipeline = (request: ImageGenerationRequest): Pipeline => + compose('imageGenerationSubRequest', [ + emitHostedImageGeneration, + prepareHostedImageGeneration, + ...openaiImagesModelStages({ operation: request.action === 'edit' ? 'edits' : 'generations' }, [retryRateLimitedImages, callOpenAIImagesUpstream]), + ]); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts new file mode 100644 index 0000000000..5a6810a795 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts @@ -0,0 +1,38 @@ +import type { Fields } from './facts.ts'; +import { prepareImageRequest } from './request.ts'; +import type { GatewayServices } from '../../../../pipeline/services.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { openaiImagesRequestWantsStream } from '@floway-dev/protocols/openai-images'; + +export const prepareHostedImageGeneration = defineStage< + Fields<'request.imageGeneration.canonical'>, + Fields<'request.imageGeneration.canonical' | 'request.openaiImages.canonical' | 'serve.model' | 'ingress.openaiImages.wantsStream'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'prepareHostedImageGeneration', + through: { + request: { needs: ['request.imageGeneration.canonical'], consumes: [], provides: ['request.openaiImages.canonical', 'serve.model', 'ingress.openaiImages.wantsStream'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status', 'response.http.headers', 'response.usage.billable'] }, + execute: async (facts, next, use) => { + let request; + try { + request = await prepareImageRequest(facts['request.imageGeneration.canonical']); + } catch (error) { + use.gateway.dump?.failed(error); + return move({ + ...facts, 'response.openaiImages.canonical': { status: 502, message: error instanceof Error ? error.message : String(error) }, + 'response.openaiImages.streamedUsage': null, 'response.http.status': 502, 'response.http.headers': [], 'response.usage.billable': [], + }); + } + return await next({ + ...facts, 'request.openaiImages.canonical': move(request), + 'serve.model': facts['request.imageGeneration.canonical'].config.model, + 'ingress.openaiImages.wantsStream': openaiImagesRequestWantsStream(request), + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts new file mode 100644 index 0000000000..6c24f7c00f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts @@ -0,0 +1,186 @@ +import { getImageProcessor } from '@floway-dev/platform'; +import { encodeHex } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIImagesEditsRequest, CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; + +// gpt-image-* `/images/edits` accepts only these input image mimetypes; a live +// Azure probe confirmed png/jpeg/webp succeed while gif is rejected with +// `unsupported_file_mimetype`. Native OpenAI Responses accepts the same GIF and +// re-encodes it before editing, so the dispatcher mirrors that behavior through the +// platform image processor. Common aliases are folded onto the backend form. +type EditMime = 'image/png' | 'image/jpeg' | 'image/webp'; + +const EDIT_MIME_ALIASES: Record = { + 'image/jpg': 'image/jpeg', + 'image/pjpeg': 'image/jpeg', + 'image/x-png': 'image/png', +}; +// The canonical edit-supported mimetype for a source, or null when the +// standalone endpoint requires local WebP transcoding first. +const editSupportedMime = (mime: string): EditMime | null => { + const canonical = EDIT_MIME_ALIASES[mime] ?? mime; + return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' + ? canonical + : null; +}; + +const editFileExt = (mime: EditMime): string => + mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; + +export interface ImageBackendConfig { + model: string; + size?: string; + quality?: string; + output_format?: 'png' | 'jpeg'; + background?: 'transparent' | 'opaque' | 'auto'; + moderation?: 'auto' | 'low'; + output_compression?: number; + // When > 0, the backend call is issued with `stream:true` and each + // progressively-rendered preview the backend emits is relayed as a native + // `image_generation_call.partial_image` frame. When 0/absent the backend + // is called non-streaming and no preview frames are produced. + partial_images?: number; + input_fidelity?: 'high' | 'low'; + action: 'generate' | 'edit' | 'auto'; +} + +export interface ImageSource { + bytes: Uint8Array; + mimeType: string; +} +interface PreparedImageSource extends ImageSource { mimeType: EditMime } + +export interface ImageGenerationRequest { + prompt: string; + action: 'generate' | 'edit'; + config: ImageBackendConfig & { mask?: ImageSource }; + sources: readonly ImageSource[]; +} + +const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { + const keyBySource = new Map>(); + const preparedByContent = new Map>(); + return await Promise.all(sources.map(async source => { + const mimeType = editSupportedMime(source.mimeType); + if (mimeType !== null) return { bytes: source.bytes, mimeType }; + + let keyPromise = keyBySource.get(source); + if (keyPromise === undefined) { + keyPromise = crypto.subtle.digest('SHA-256', source.bytes).then(buffer => { + const digest = encodeHex(new Uint8Array(buffer)); + return `${source.mimeType}\u0000${digest}`; + }); + keyBySource.set(source, keyPromise); + } + const key = await keyPromise; + + let prepared = preparedByContent.get(key); + if (prepared === undefined) { + // Native OpenAI Responses accepts formats such as GIF through its multimodal + // preprocessing, while the standalone edits endpoint accepts only + // PNG/JPEG/WebP. Re-encode locally to preserve the hosted-tool behavior. + // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 + prepared = getImageProcessor().compressToWebp(source.bytes, null).then(encoded => { + const bytes = new Uint8Array(encoded); + return { bytes, mimeType: 'image/webp' } satisfies PreparedImageSource; + }); + preparedByContent.set(key, prepared); + } + return await prepared; + })); +}; + +const prepareEditRequest = async ( + sources: readonly ImageSource[], + config: ImageBackendConfig & { mask?: ImageSource }, +): Promise<{ sources: readonly PreparedImageSource[]; mask?: PreparedImageSource }> => { + const originals = [...sources]; + if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); + const prepared = await prepareEditSources(originals); + return { + sources: prepared.slice(0, sources.length), + ...(config.mask === undefined ? {} : { mask: prepared[originals.indexOf(config.mask)]! }), + }; +}; + +const buildGenerationsBody = (prompt: string, config: ImageBackendConfig, stream: boolean): Record => ({ + prompt, + // Public OpenAI Responses tool config forbids `n`, but the private standalone + // backend call always requests a single image, mirroring Azure's + // single-image OpenAI Responses behavior. + n: 1, + // `response_format` is intentionally not sent: gpt-image-* always returns + // base64 (`data[0].b64_json`) and rejects `response_format`, so the inline + // extraction below reads `b64_json` directly. + ...(config.size !== undefined ? { size: config.size } : {}), + ...(config.quality !== undefined ? { quality: config.quality } : {}), + ...(config.output_format !== undefined ? { output_format: config.output_format } : {}), + ...(config.background !== undefined ? { background: config.background } : {}), + ...(config.moderation !== undefined ? { moderation: config.moderation } : {}), + ...(config.output_compression !== undefined ? { output_compression: config.output_compression } : {}), + ...(stream ? { stream: true, partial_images: config.partial_images } : {}), +}); + +const buildEditsRequest = ( + prompt: string, + config: ImageBackendConfig, + sources: readonly PreparedImageSource[], + mask: PreparedImageSource | undefined, + stream: boolean, +): CanonicalOpenAIImagesEditsRequest => { + const parameters: Record = { + prompt, + n: 1, + ...(config.size === undefined ? {} : { size: config.size }), + ...(config.quality === undefined ? {} : { quality: config.quality }), + ...(config.output_format === undefined ? {} : { output_format: config.output_format }), + ...(config.background === undefined ? {} : { background: config.background }), + ...(config.moderation === undefined ? {} : { moderation: config.moderation }), + ...(config.output_compression === undefined ? {} : { output_compression: config.output_compression }), + ...(config.input_fidelity === undefined ? {} : { input_fidelity: config.input_fidelity }), + ...(stream ? { stream: true, partial_images: config.partial_images } : {}), + }; + const images = sources.map((source, index) => ({ + kind: 'file' as const, + file: { bytes: source.bytes, fileName: `image_${index}.${editFileExt(source.mimeType)}`, mediaType: source.mimeType }, + })); + const maskFile = mask === undefined + ? undefined + : { bytes: mask.bytes, fileName: `mask.${editFileExt(mask.mimeType)}`, mediaType: mask.mimeType }; + return { + operation: 'edits', + images, + ...(maskFile === undefined ? {} : { mask: { kind: 'file' as const, file: maskFile } }), + parameters, + }; +}; + +export const prepareImageRequest = async (request: ImageGenerationRequest): Promise => { + const { prompt, config } = request; + const stream = (config.partial_images ?? 0) > 0; + if (request.action === 'generate') return { operation: 'generations', parameters: buildGenerationsBody(prompt, config, stream) }; + const prepared = await prepareEditRequest(request.sources, config); + return buildEditsRequest(prompt, config, prepared.sources, prepared.mask, stream); +}; + +export const portableImageRequest = ( + prompt: string, + action: 'generate' | 'edit', + config: ImageBackendConfig & { mask?: { bytes: ArrayBuffer; mimeType: string } }, + sources: readonly { bytes: ArrayBuffer; mimeType: string }[], +): ImageGenerationRequest => { + const converted = new Map(); + const convert = (source: { bytes: ArrayBuffer; mimeType: string }): ImageSource => { + let portable = converted.get(source); + if (portable === undefined) { + portable = { bytes: new Uint8Array(source.bytes), mimeType: source.mimeType }; + converted.set(source, portable); + } + return portable; + }; + const { mask, ...parameters } = config; + return { + prompt, action, + config: { ...parameters, ...(mask === undefined ? {} : { mask: convert(mask) }) }, + sources: sources.map(convert), + }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts new file mode 100644 index 0000000000..cb6041416d --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts @@ -0,0 +1,133 @@ +import type { HostedToolOutputItem, HostedToolTerminal } from '../types.ts'; +import type { OpenAIImagesStreamEvent } from '@floway-dev/protocols/openai-images'; +import type { OpenAIResponsesOutputImageGenerationCall } from '@floway-dev/protocols/openai-responses'; + +export type ImageError = { type: string; code: string; message: string; retryable: boolean }; + +// Server-resolved tool config echoed by the backend on both the partial_image +// frames and the final result (`background:"auto"` becomes the concrete value +// the server picked, etc.). Read straight off the backend rather than inferred +// from the request, so what we surface matches what was actually rendered. +interface EchoFields { + output_format?: 'png' | 'jpeg'; + quality?: 'low' | 'medium' | 'high'; + background?: 'transparent' | 'opaque'; + size?: string; +} + +export type ImageOutcome = + | { ok: true; b64: string; echo: EchoFields } + | { ok: false; error: ImageError }; + +// Project the server-resolved echo fields out of a backend payload (a response +// JSON body or an SSE event). Each field is validated against the public enum +// so a surprising backend value is dropped rather than echoed verbatim. +export const extractEcho = (source: unknown): EchoFields => { + if (source === null || typeof source !== 'object') return {}; + const s = source as Record; + const echo: EchoFields = {}; + if (s.output_format === 'png' || s.output_format === 'jpeg') echo.output_format = s.output_format; + if (s.quality === 'low' || s.quality === 'medium' || s.quality === 'high') echo.quality = s.quality; + if (s.background === 'transparent' || s.background === 'opaque') echo.background = s.background; + if (typeof s.size === 'string') echo.size = s.size; + return echo; +}; + +const RETRYABLE_IMAGE_ERROR_CODES = new Set([ + 'EngineOverloaded', 'server_error', 'image_generation_server_error', 'image_generation_failed', +]); + +export const isRetryableImageError = (code: string, type?: string): boolean => + RETRYABLE_IMAGE_ERROR_CODES.has(code) || (type !== undefined && RETRYABLE_IMAGE_ERROR_CODES.has(type)); + +export const errorFromBody = (body: string, status: number): { type?: string; code: string; message: string } => { + try { + const parsed = JSON.parse(body) as { error?: { message?: unknown; code?: unknown; type?: unknown } }; + const err = parsed.error; + if (err !== undefined && err !== null) { + return { + ...(typeof err.type === 'string' ? { type: err.type } : {}), + message: typeof err.message === 'string' ? err.message : `Image backend returned HTTP ${status}`, + code: typeof err.code === 'string' ? err.code : `upstream_${status}`, + }; + } + } catch (e) { + if (!(e instanceof SyntaxError)) throw e; + } + return { message: `Image backend returned HTTP ${status}`, code: `upstream_${status}` }; +}; + +// Build the completed/failed `image_generation_call` output item plus its +// closing events. On success the final bytes ride `output_item.done.item.result` +// and a `response.image_generation_call.completed` closes the item; on failure +// neither `.completed` nor any `.partial_image` is emitted — only the failed +// `output_item.done`. +// +// `revised_prompt` is set to the orchestrator's prompt: the standalone images +// backend does no prompt rewriting and returns no `revised_prompt`, and the +// orchestrator's emitted prompt IS already its refined rewrite (it plays the +// role Azure's native flow gives the orchestrator), so it is the faithful +// source for this field. +export const imageTerminal = ( + prompt: string, + action: 'generate' | 'edit', + outcome: ImageOutcome, +): HostedToolTerminal => { + if (!outcome.ok) { + const item: HostedToolOutputItem & Omit = { + type: 'image_generation_call', + status: 'failed', + revised_prompt: prompt, + error: { message: outcome.error.message, code: outcome.error.code, type: outcome.error.type }, + }; + return { item, endEvents: [] }; + } + + const item: HostedToolOutputItem & Omit = { + type: 'image_generation_call', + status: 'completed', + action, + result: outcome.b64, + revised_prompt: prompt, + ...outcome.echo, + }; + return { item, endEvents: [{ type: 'response.image_generation_call.completed' }] }; +}; + +// Native image events become the hosted lifecycle without transport reframing. +// The generations and edits endpoints use distinct event prefixes +// (`image_generation.*` vs `image_edit.*`); only the suffix is matched here. +type ImageStreamSignal = + | { kind: 'partial'; index: number; b64: string; echo: EchoFields } + | { kind: 'completed'; b64: string | undefined; usage: unknown; echo: EchoFields } + | { kind: 'error'; error: ImageError } + | null; + +export const projectImageStreamEvent = (evt: OpenAIImagesStreamEvent): ImageStreamSignal => { + const type = typeof evt.type === 'string' ? evt.type : ''; + if (type.endsWith('.partial_image')) { + return { + kind: 'partial', + index: typeof evt.partial_image_index === 'number' ? evt.partial_image_index : 0, + b64: typeof evt.b64_json === 'string' ? evt.b64_json : '', + echo: extractEcho(evt), + }; + } + if (type.endsWith('.completed')) { + return { kind: 'completed', b64: typeof evt.b64_json === 'string' ? evt.b64_json : undefined, usage: evt.usage, echo: extractEcho(evt) }; + } + if (type === 'error') { + const err = evt.error as { message?: unknown; code?: unknown; type?: unknown } | undefined; + const code = typeof err?.code === 'string' ? err.code : 'server_error'; + const errType = typeof err?.type === 'string' ? err.type : 'image_generation_error'; + return { + kind: 'error', + error: { type: errType, code, message: typeof err?.message === 'string' ? err.message : 'Image backend stream reported an error.', retryable: isRetryableImageError(code, errType) }, + }; + } + return null; +}; + +export const serverError = (error: unknown): ImageError => ({ + type: 'image_generation_error', message: error instanceof Error ? error.message : String(error), code: 'server_error', retryable: true, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts new file mode 100644 index 0000000000..d511868cda --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts @@ -0,0 +1,78 @@ +import { sleep } from '../../../../../shared/sleep.ts'; +import type { Fields } from '../../../../openai-images/facts.ts'; +import type { BillableEntity } from '../../../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../../../pipeline/serve.ts'; +import type { GatewayServices } from '../../../../pipeline/services.ts'; +import { defineStage, move, defer, type Deferred } from '@floway-dev/pipeline'; + +// 60s cap matches the per-minute refill window of Azure TPM/RPM and +// openai.com tier image quotas — same clamp openai-python applies in +// [`_calculate_retry_timeout`](https://github.com/openai/openai-python/blob/d76d8c11c1da9f97aa8a0aaee8ccd44d2bc8f5e7/src/openai/_base_client.py#L789). +const RETRY_CAP_MS = 60_000; +const MAX_RATE_LIMIT_RETRIES = 2; + +// Header priority matches openai-python's `_parse_retry_after_header` with +// Azure's `x-ms-retry-after-ms` alias added. Treats <= 0 as "no hint" so the +// gpt-image-1 `retry-after: 0.0` quirk falls back to backoff instead of +// pretending the quota is free. +export const parseRetryAfterMs = (headers: Headers): number | null => { + for (const name of ['retry-after-ms', 'x-ms-retry-after-ms']) { + const raw = headers.get(name); + if (raw === null) continue; + const ms = Number(raw); + if (Number.isFinite(ms) && ms > 0) return ms; + } + const ra = headers.get('retry-after'); + if (ra !== null) { + const seconds = Number(ra); + if (Number.isFinite(seconds) && seconds > 0) return seconds * 1000; + const httpDateMs = Date.parse(ra); + if (!Number.isNaN(httpDateMs)) { + const delta = httpDateMs - Date.now(); + if (delta > 0) return delta; + } + } + return null; +}; + +export const retryRateLimitedImages = defineStage< + Fields<'request.openaiImages.canonical' | 'route.attempt'>, + Fields<'request.openaiImages.canonical' | 'route.attempt'>, + Fields<'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable' | 'response.openaiImages.streamedUsage'>, + Fields<'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable' | 'response.openaiImages.streamedUsage'>, + GatewayServices +>({ + name: 'retryRateLimitedImages', + through: { + request: { needs: ['request.openaiImages.canonical', 'route.attempt'], consumes: [], provides: [] }, + response: { + needs: ['response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable', 'response.openaiImages.streamedUsage'], + consumes: ['response.http.body'], + provides: ['response.http.body', 'response.usage.billable', 'response.openaiImages.streamedUsage'], + }, + }, + execute: async (facts, next, use) => { + const prior: BillableEntity[] = []; + for (let retry = 0; ; retry++) { + const back = await next(facts); + if (back['response.http.status'] !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) { + if (prior.length === 0) return back; + const pending = back['response.openaiImages.streamedUsage'] as Deferred | null; + const calls = move([...prior]); + return { + ...back, + 'response.usage.billable': move([...calls, ...back['response.usage.billable']]), + 'response.openaiImages.streamedUsage': pending === null ? null + : move(defer(pending.then(outcome => ({ ...outcome, billable: move([...calls, ...outcome.billable]) })))), + }; + } + prior.push(...back['response.usage.billable']); + const base = 1000 * 2 ** retry; + const backoffMs = base + Math.random() * base * 0.25; + const headers = new Headers(back['response.http.headers'].map(([name, value]): [string, string] => [name, value])); + await sleep(Math.min(parseRetryAfterMs(headers) ?? backoffMs, RETRY_CAP_MS), use.gateway.abortSignal); + use.gateway.attempt.timing.upstreamCallStartedAt = null; + use.gateway.attempt.timing.firstOutputTokenAt = null; + } + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/services.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/services.ts deleted file mode 100644 index d5d382c5fe..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/services.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { BillableEntity } from '../../../../pipeline/facts.ts'; -import type { GatewayServices } from '../../../../pipeline/services.ts'; -import type { PerformanceTelemetryContext } from '../../../../shared/telemetry/performance.ts'; -import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../types.ts'; - -/** How the ending reports what it did, once it knows. The identity and the counts arrive with - * the backend's own response, long after this run handed up. */ -export type SettleImageCall = (billable: readonly BillableEntity[], failed: boolean, telemetry: PerformanceTelemetryContext | undefined) => void; - -export type ImageCall = (settle: SettleImageCall) => AsyncGenerator; - -export interface ImageServices extends GatewayServices { readonly imageCall: ImageCall } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts index ffcb700012..405759e382 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts @@ -815,16 +815,18 @@ export async function* materializeHostedToolItems( for (const d of dispatched) { for (const { slot, outputIndex } of d.slots) { const lifecycle = slot.run(); - let step = await lifecycle.next(); - while (!step.done) { - yield stampHostedToolEvent(merge, outputIndex, slot.id, step.value); - step = await lifecycle.next(); + try { + let step = await lifecycle.next(); + while (!step.done) { + yield stampHostedToolEvent(merge, outputIndex, slot.id, step.value); + step = await lifecycle.next(); + } + // Register dispatcher state before item.done makes its stored row reusable. + store.registerPrivatePayload(slot.id, step.value.privatePayload); + yield* hostedToolEndFrames(merge, outputIndex, slot, step.value); + } finally { + await lifecycle.return(undefined as never); } - // Register private dispatcher state under the emitted item id so output - // persistence captures it and replay-side `transformItems` can restore - // it on the next loop turn. - store.registerPrivatePayload(slot.id, step.value.privatePayload); - yield* hostedToolEndFrames(merge, outputIndex, slot, step.value); } } } diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index 5bce6d5af8..2060a898c2 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -6,20 +6,22 @@ import { isFailure } from '../pipeline/facts.ts'; import { failover } from '../pipeline/failover.ts'; import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; -import { compose, type Pipeline } from '@floway-dev/pipeline'; +import { compose, type Pipeline, type Stage } from '@floway-dev/pipeline'; import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; export const openaiImagesServePipeline = (request: CanonicalOpenAIImagesRequest): Pipeline => - compose('openaiImagesServe', [ - emitOpenAIImages, - writeSettlement( - handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), - handedUp => (handedUp as { 'response.openaiImages.streamedUsage'?: unknown })['response.openaiImages.streamedUsage'] !== null, - ), - resolveCandidates(narrowing(request)), - failover({ - failed: handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), - owns: ['response.http.body'], - }), - callOpenAIImagesUpstream, - ]); + compose('openaiImagesServe', [emitOpenAIImages, ...openaiImagesModelStages(request)]); + +export const openaiImagesModelStages = (request: Pick, dispatchStages: readonly Stage[] = [callOpenAIImagesUpstream]): readonly Stage[] => [ + writeSettlement( + handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), + handedUp => (handedUp as { 'response.openaiImages.streamedUsage'?: unknown })['response.openaiImages.streamedUsage'] !== null, + ), + resolveCandidates(narrowing(request)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), + owns: ['response.http.body'], + streamedUsage: 'response.openaiImages.streamedUsage', + }), + ...dispatchStages, +]; diff --git a/packages/gateway/src/data-plane/openai-images/target.ts b/packages/gateway/src/data-plane/openai-images/target.ts index bc83782a13..fe140e8d70 100644 --- a/packages/gateway/src/data-plane/openai-images/target.ts +++ b/packages/gateway/src/data-plane/openai-images/target.ts @@ -11,7 +11,7 @@ const ENDPOINT = { /** A candidate that cannot serve *this* request is not a candidate. One family covers two * endpoints and an upstream may expose either without the other, so which one is asked for is * what narrows the list. */ -export const narrowing = (request: CanonicalOpenAIImagesRequest) => ({ +export const narrowing = (request: Pick) => ({ kind: 'image' as const, reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined ? `the upstream does not expose the OpenAI Images ${request.operation} endpoint` From 2972be2c59d5db627969aa859097eaa9def41d4a Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 04:03:51 +0800 Subject: [PATCH 116/156] fix(chat): record the final client protocol projection Apply usage selection, native terminator removal and Anthropic citation wire projection before recording streaming client frames. Keep nonstream collectors on canonical events and make transport framing a pure map. Preserve narrowed protocol frame types through recordStream. Compare selected decoded client streams with actual SSE for Chat Completions usage selection, DONE and normal EOF, Anthropic citations, and Gemini termination. Focused gateway and protocol regressions plus gateway typecheck and lint pass. --- .../data-plane/chat/client-streams_test.ts | 102 ++++++++++++++++++ .../chat/anthropic-messages/emit.ts | 38 +++---- .../chat/gemini-generate-content/emit.ts | 32 +++--- .../chat/openai-chat-completions/emit.ts | 34 +++--- packages/gateway/src/dump/run-sink.ts | 2 +- .../protocols/src/anthropic-messages/index.ts | 2 +- .../src/anthropic-messages/to-sse.ts | 2 +- .../src/openai-chat-completions/to-sse.ts | 6 +- 8 files changed, 164 insertions(+), 54 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/client-streams_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts b/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts new file mode 100644 index 0000000000..2920ec6de7 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts @@ -0,0 +1,102 @@ +import { expect, test } from 'vitest'; + +import { emitAnthropicMessages } from '../../../src/data-plane/chat/anthropic-messages/emit.ts'; +import { emitGeminiGenerateContent } from '../../../src/data-plane/chat/gemini-generate-content/emit.ts'; +import { emitOpenAIChatCompletions } from '../../../src/data-plane/chat/openai-chat-completions/emit.ts'; +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { openRunDump } from '../../../src/dump/run-sink.ts'; +import type { ApiKey } from '../../../src/repo/types.ts'; +import { eventsOf, installDumpStubs } from '../../dump/test-fixtures.ts'; +import { flushBackground, trackBackground } from '../../test-utils/background-tracker.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, createRunReader, defineStage, move, run, type DumpEvent } from '@floway-dev/pipeline'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; + +const key: ApiKey = { id: 'client-key', userId: 1, name: 'Client key', key: 'client-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; +const usage: OpenAIChatCompletionsStreamEvent = { id: 'chat', object: 'chat.completion.chunk', created: 1, model: 'model', choices: [], usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 } }; +const chat: OpenAIChatCompletionsStreamEvent = { id: 'chat', object: 'chat.completion.chunk', created: 1, model: 'model', choices: [{ index: 0, delta: { content: 'answer' }, finish_reason: 'stop' }] }; +const citation: AnthropicMessagesStreamEvent = { + type: 'content_block_start', index: 0, + content_block: { + type: 'text', text: 'answer', + citations: [{ type: 'search_result_location', url: 'https://example.com/source', title: 'Source', search_result_index: 0, start_block_index: 0, end_block_index: 1 }], + }, +}; +const gemini: GeminiGenerateContentStreamEvent = { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'answer' }] }, finishReason: 'STOP' }] }; + +const cases = [ + ...[false, true].flatMap(includeUsage => [false, true].map(done => ({ + name: `Chat Completions ${includeUsage ? 'included' : 'hidden'} usage with ${done ? 'DONE' : 'normal EOF'}`, + emit: emitOpenAIChatCompletions, + entry: { 'ingress.chat.openaiChatCompletions.wantsStream': true, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': includeUsage }, + response: 'response.chat.openaiChatCompletions', rendered: 'response.chat.openaiChatCompletions.rendered', + upstream: [eventFrame(chat), eventFrame(usage), ...(done ? [doneFrame()] : [])], + verify: (events: unknown[]) => { + expect(events.some(event => typeof event === 'object' && event !== null && 'usage' in event)).toBe(includeUsage); + expect(events.includes('[DONE]')).toBe(done); + }, + }))), + { + name: 'Anthropic citations and transport terminator', emit: emitAnthropicMessages, + entry: { 'ingress.chat.anthropicMessages.wantsStream': true }, + response: 'response.chat.anthropicMessages', rendered: 'response.chat.anthropicMessages.rendered', + upstream: [eventFrame(citation), eventFrame({ type: 'content_block_stop', index: 0 }), eventFrame({ type: 'message_stop' }), doneFrame()], + verify: (events: unknown[]) => { + expect(events).not.toContain('[DONE]'); + expect(events).toContainEqual({ ...citation, index: 1, content_block: { type: 'text', text: 'answer', citations: [{ type: 'search_result_location', source: 'https://example.com/source', title: 'Source', search_result_index: 0, start_block_index: 0, end_block_index: 1 }] } }); + }, + }, + { + name: 'Gemini transport terminator', emit: emitGeminiGenerateContent, + entry: { 'ingress.chat.geminiGenerateContent.wantsStream': true }, + response: 'response.chat.geminiGenerateContent', rendered: 'response.chat.geminiGenerateContent.rendered', + upstream: [eventFrame(gemini), doneFrame()], + verify: (events: unknown[]) => { expect(events).not.toContain('[DONE]'); expect(events).toMatchObject([gemini]); }, + }, +]; + +for (const fixture of cases) { + test(`selected recorded client protocol frames equal actual SSE for ${fixture.name}`, async () => { + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(key, { method: 'POST', path: '/v1/chat', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + if (dump === null) throw new Error('retained client request has no run recording'); + const dial = defineStage, Record>({ + name: 'scriptedDial', return: { provides: [fixture.response, 'response.http.headers'] }, + execute: async facts => move({ + ...facts, 'response.http.headers': [], + [fixture.response]: { kind: 'stream', frames: { async *[Symbol.asyncIterator]() { yield* fixture.upstream; } } }, + }), + }); + const outcome = await run(compose, Record>('clientStream', [fixture.emit, dial]), move(fixture.entry), { gateway: mockChatGatewayCtx({ dump, wantsStream: true }), dump: dump.sink }); + const wire: unknown[] = []; + for await (const frame of outcome.facts[fixture.rendered] as AsyncIterable) wire.push(frame.data === '[DONE]' ? frame.data : JSON.parse(frame.data)); + await outcome.drain(); + dump.finalize(200, 0); + await flushBackground(); + const read = createRunReader(); + const streams = new Map[]>(); + let client: number | null = null; + const record = stubs.stored[0]?.record; + if (record === undefined) throw new Error('completed client stream has no persisted record'); + for (const item of eventsOf(record)) { + const event = item as unknown as DumpEvent; + const decoded = read(event); + if (decoded?.facts && 'response.chat.clientFrames' in decoded.facts) client = (decoded.facts['response.chat.clientFrames'] as { stream: number }).stream; + if (event.type === 'stream.frame') { + const frames = streams.get(event.streamId) ?? []; + frames.push(...decoded?.frames as ProtocolFrame[]); + streams.set(event.streamId, frames); + } + } + const frames = client === null ? [] : streams.get(client) ?? []; + expect(frames.length).toBeGreaterThan(0); + const recorded = frames.map(frame => frame.type === 'done' ? '[DONE]' : frame.event); + expect(recorded).toEqual(wire); + expect(eventsOf(record).some(event => event.type === 'stream.end' && event.streamId === client)).toBe(true); + expect(record.meta.error).toBeNull(); + fixture.verify(recorded); + }); +} diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts index b76d7c366e..f3a8f1ee0a 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts @@ -7,8 +7,8 @@ import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import { collectAnthropicMessagesProtocolEventsToResult, anthropicMessagesProtocolFrameToSSEFrame, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { sseFrame, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; +import { collectAnthropicMessagesProtocolEventsToResult, anthropicMessagesEventToSsePayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; /** * The outermost edge. An Anthropic Messages answer is always a stream by the time it reaches here — @@ -69,14 +69,12 @@ export const emitAnthropicMessages = defineStage< // carrying it comes back to the upstream that issued it. This is the other half of the // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. - const frames = recordStream( - wrapAnthropicMessagesAffinityEgress( - answer.frames as AsyncIterable>, - affinityEgressOptions(use.gateway), - ), - use.gateway.dump, + const egress = wrapAnthropicMessagesAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), ); if (!back['ingress.chat.anthropicMessages.wantsStream']) { + const frames = recordStream(egress, use.gateway.dump); return { ...rest, 'response.chat.clientFrames': move(frames), @@ -87,6 +85,7 @@ export const emitAnthropicMessages = defineStage< 'response.http.status': 200, }; } + const frames = recordStream(clientFrames(egress), use.gateway.dump); return { ...rest, 'response.chat.clientFrames': move(frames), @@ -97,22 +96,25 @@ export const emitAnthropicMessages = defineStage< }, }); -/** Anthropic names its own SSE events, and every frame that has one is the client's. Which - * frames have an SSE form at all is the protocol's to say, and it says so by writing no - * frame — a stream terminator is an OpenAI Chat Completions idea and there is nothing to send for - * it here. */ -const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ - // The frames the client reads are a reframing of the ones the record holds, so this key - // points at that same stream rather than at nothing. - ...streamReferenceOf(frames), +type ClientEvent = ReturnType; + +// SSE citations have their wire spelling before recording; the protocol's synthetic done +// frame has no Anthropic client form. Nonstream reassembly retains its canonical events. +const clientFrames = (frames: AsyncIterable>): AsyncIterable> => ({ [Symbol.asyncIterator]: () => (async function* () { for await (const frame of frames) { - const written = anthropicMessagesProtocolFrameToSSEFrame(frame); - if (written !== null) yield written; + if (frame.type === 'event') yield eventFrame(anthropicMessagesEventToSsePayload(frame.event)); } })(), }); +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ + ...streamReferenceOf(frames), + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) yield sseFrame(JSON.stringify(frame.event), frame.event.type); + })(), +}); + /** What this protocol writes on an idle connection. Anthropic defines a `ping` event and * clients read it, so an SSE comment — invisible by design — is not the same wire byte. * The route that serves this chain hands it to the seam alongside the frames. */ diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts index 780f4e492c..75cc2ec8d4 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts @@ -7,8 +7,8 @@ import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; -import { collectGeminiGenerateContentProtocolEventsToResult, geminiGenerateContentProtocolFrameToSSEFrame, type GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; +import { sseFrame, type EventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { collectGeminiGenerateContentProtocolEventsToResult, type GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; /** * The outermost edge. A Gemini generateContent answer is always a stream by the time it reaches @@ -72,14 +72,12 @@ export const emitGeminiGenerateContent = defineStage< // carrying it comes back to the upstream that issued it. This is the other half of the // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. - const frames = recordStream( - wrapGeminiGenerateContentAffinityEgress( - answer.frames as AsyncIterable>, - affinityEgressOptions(use.gateway), - ), - use.gateway.dump, + const egress = wrapGeminiGenerateContentAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), ); if (!back['ingress.chat.geminiGenerateContent.wantsStream']) { + const frames = recordStream(egress, use.gateway.dump); return { ...rest, 'response.chat.clientFrames': move(frames), @@ -90,6 +88,7 @@ export const emitGeminiGenerateContent = defineStage< 'response.http.status': 200, }; } + const frames = recordStream(clientFrames(egress), use.gateway.dump); return { ...rest, 'response.chat.clientFrames': move(frames), @@ -100,16 +99,15 @@ export const emitGeminiGenerateContent = defineStage< }, }); -/** Gemini generateContent streams one JSON object per `data:` line and has no sentinel to write, which the - * protocol says by writing no frame at all for the one that ends the stream. */ -const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ - // The frames the client reads are a reframing of the ones the record holds, so this key - // points at that same stream rather than at nothing. +const clientFrames = (frames: AsyncIterable>): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) if (frame.type === 'event') yield frame; + })(), +}); + +const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ ...streamReferenceOf(frames), [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) { - const written = geminiGenerateContentProtocolFrameToSSEFrame(frame); - if (written !== null) yield written; - } + for await (const frame of frames) yield sseFrame(JSON.stringify(frame.event)); })(), }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts index 335ba906e7..2c6875d47e 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts @@ -6,7 +6,7 @@ import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; +import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectOpenAIChatCompletionsProtocolEventsToResult, openaiChatCompletionsProtocolFrameToSSEFrame, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; /** @@ -71,11 +71,13 @@ export const emitOpenAIChatCompletions = defineStage< // carrying it comes back to the upstream that issued it. This is the other half of the // affinity the resolver read on the way down, and it has to sit here because it rewrites // the frames — below the fold, and there would be nothing left to rewrite. + const egress = wrapOpenAIChatCompletionsAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ); const frames = recordStream( - wrapOpenAIChatCompletionsAffinityEgress( - answer.frames as AsyncIterable>, - affinityEgressOptions(use.gateway), - ), + back['ingress.chat.openaiChatCompletions.wantsStream'] + ? clientFrames(egress, back['ingress.chat.openaiChatCompletions.wantsUsageChunk']) : egress, use.gateway.dump, ); if (!back['ingress.chat.openaiChatCompletions.wantsStream']) { @@ -93,26 +95,30 @@ export const emitOpenAIChatCompletions = defineStage< ...rest, 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, - 'response.chat.openaiChatCompletions.rendered': move(renderSSE(frames, back['ingress.chat.openaiChatCompletions.wantsUsageChunk'])), + 'response.chat.openaiChatCompletions.rendered': move(renderSSE(frames)), 'response.http.status': 200, }; }, }); -/** Metering asks the upstream for a usage chunk on every streaming turn; whether the client - * is shown it is the client's own question. The protocol owns which frame that is, and says - * so by writing no frame at all. */ +// Metering reads every usage chunk below this edge. A streaming client opts into seeing it, +// so that protocol selection precedes the record of what the client is served. +const clientFrames = (frames: AsyncIterable>, includeUsageChunk: boolean): AsyncIterable> => includeUsageChunk ? frames : ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + if (frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; + yield frame; + } + })(), +}); + const renderSSE = ( frames: AsyncIterable>, - includeUsageChunk: boolean, ): AsyncIterable => ({ // The frames the client reads are a reframing of the ones the record holds, so this key // points at that same stream rather than at nothing. ...streamReferenceOf(frames), [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) { - const written = openaiChatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk }); - if (written !== null) yield written; - } + for await (const frame of frames) yield openaiChatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk: true }); })(), }); diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 6f1269034a..963a5eeab0 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -275,7 +275,7 @@ export interface StreamRecording { * family whose stream is bare protocol events says how one becomes a frame, because the record * cannot guess and a cast would be it guessing. */ -export function recordStream(stream: AsyncIterable>, dump: RunDump | null): AsyncIterable>; +export function recordStream>(stream: AsyncIterable, dump: RunDump | null): AsyncIterable; export function recordStream(stream: AsyncIterable, dump: RunDump | null, asFrame: (value: T) => ProtocolFrame): AsyncIterable; export function recordStream( stream: AsyncIterable, diff --git a/packages/protocols/src/anthropic-messages/index.ts b/packages/protocols/src/anthropic-messages/index.ts index f61a05c576..082ca4303d 100644 --- a/packages/protocols/src/anthropic-messages/index.ts +++ b/packages/protocols/src/anthropic-messages/index.ts @@ -397,5 +397,5 @@ export const parseAnthropicBetaHeader = (raw: string | null | undefined): readon export { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, collectAnthropicMessagesProtocolEventsToResult } from './to-result.ts'; export { generateAnthropicId } from './id.ts'; export { reassembleAnthropicMessagesEvents } from './reassemble.ts'; -export { anthropicMessagesProtocolFrameToSSEFrame } from './to-sse.ts'; +export { anthropicMessagesEventToSsePayload, anthropicMessagesProtocolFrameToSSEFrame } from './to-sse.ts'; export { PROMPT_TOO_LONG_MESSAGE, buildPromptTooLongBody } from './context-window-error.ts'; diff --git a/packages/protocols/src/anthropic-messages/to-sse.ts b/packages/protocols/src/anthropic-messages/to-sse.ts index 378b133a1f..8275695478 100644 --- a/packages/protocols/src/anthropic-messages/to-sse.ts +++ b/packages/protocols/src/anthropic-messages/to-sse.ts @@ -51,7 +51,7 @@ const citationToSsePayload = (citation: AnthropicMessagesTextCitation): Anthropi } : citation; -const anthropicMessagesEventToSsePayload = (event: AnthropicMessagesStreamEvent): AnthropicMessagesSseEventPayload => { +export const anthropicMessagesEventToSsePayload = (event: AnthropicMessagesStreamEvent): AnthropicMessagesSseEventPayload => { if (event.type === 'content_block_start') { const { content_block } = event; if (content_block.type !== 'text' || !content_block.citations) return event as AnthropicMessagesSseEventPayload; diff --git a/packages/protocols/src/openai-chat-completions/to-sse.ts b/packages/protocols/src/openai-chat-completions/to-sse.ts index 632cfc37a9..47dbbd3497 100644 --- a/packages/protocols/src/openai-chat-completions/to-sse.ts +++ b/packages/protocols/src/openai-chat-completions/to-sse.ts @@ -5,8 +5,10 @@ interface OpenAIChatCompletionsSseFrameOptions { includeUsageChunk: boolean; } -export const openaiChatCompletionsProtocolFrameToSSEFrame = (frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null => { +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: { includeUsageChunk: true }): SseFrame; +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null; +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null { if (frame.type === 'done') return sseFrame('[DONE]'); if (!options.includeUsageChunk && isOpenAIUsageOnlyEventShape(frame.event)) return null; return sseFrame(JSON.stringify(frame.event)); -}; +} From c9243cbf1323fdf94d911571dd5aab05bb8b9be5 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 05:13:54 +0800 Subject: [PATCH 117/156] feat: expose chat provider operation pipelines Move Custom, Azure, Ollama and Copilot request preparation and protocol transforms into immutable provider fact stages composed with the shared HTTP terminal. Record each protocol stream production and publish actual endpoint observations before decoding can fail. Preserve Responses action steering, compact/count endpoints, Copilot raw-model selection and all existing request/response workarounds. Verify five package typechecks, targeted lint, and 678 tests across 79 provider test files. --- packages/provider-azure/src/pipelines.ts | 19 +- .../src/stages/fill-namespace-descriptions.ts | 21 ++ .../src/stages/prepare-request.ts | 35 ++- .../__tests__/stages/pipelines_test.ts | 70 +++++ packages/provider-copilot/src/chat-facts.ts | 22 ++ .../provider-copilot/src/pipeline-facts.ts | 1 + packages/provider-copilot/src/pipelines.ts | 40 ++- .../apply-top-level-cache-control.ts | 96 ++++++ .../anthropic-messages/compress-images.ts | 123 ++++++++ .../anthropic-messages/handle-speed-fast.ts | 76 +++++ .../normalize-anthropic-beta.ts | 44 +++ .../promote-thinking-display.ts | 112 +++++++ .../rewrite-context-window-error.ts | 30 ++ .../set-claude-agent-headers.ts | 71 +++++ .../anthropic-messages/set-compact-headers.ts | 139 +++++++++ .../set-initiator-header.ts | 51 ++++ .../set-interaction-id-header.ts | 46 +++ .../anthropic-messages/set-vision-header.ts | 50 +++ .../strip-cache-control-extensions.ts | 77 +++++ .../strip-eager-input-streaming.ts | 39 +++ .../src/stages/authenticate-chat.ts | 24 ++ .../abort-on-tool-argument-whitespace.ts | 76 +++++ .../attach-cache-control-markers.ts | 93 ++++++ .../compress-images.ts | 66 ++++ .../set-initiator-header.ts | 56 ++++ .../set-vision-header.ts | 33 ++ .../abort-on-tool-argument-whitespace.ts | 74 +++++ .../openai-responses/compress-images.ts | 90 ++++++ .../fill-empty-namespace-descriptions.ts | 31 ++ .../openai-responses/force-store-false.ts | 31 ++ .../openai-responses/item-id-membrane.ts | 289 ++++++++++++++++++ .../openai-responses/set-initiator-header.ts | 49 +++ .../openai-responses/set-vision-header.ts | 39 +++ .../strip-image-generation.ts | 76 +++++ .../openai-responses/strip-service-tier.ts | 36 +++ .../src/stages/prepare-chat-request.ts | 28 ++ .../src/stages/prepare-request.ts | 10 +- .../src/stages/reshape-compaction.ts | 19 ++ .../src/stages/select-chat-model.ts | 46 +++ .../__tests__/stages/pipelines_test.ts | 58 ++++ packages/provider-custom/src/pipelines.ts | 22 +- .../src/stages/prepare-request.ts | 35 ++- packages/provider-ollama/src/pipelines.ts | 14 +- .../src/stages/prepare-request.ts | 35 ++- .../__tests__/stages/decode-response_test.ts | 69 +++++ packages/provider/src/index.ts | 5 +- packages/provider/src/pipeline.ts | 48 ++- .../provider/src/stages/decode-response.ts | 58 ++++ packages/provider/src/stages/observe-call.ts | 22 ++ .../src/stages/select-responses-action.ts | 12 + packages/test-utils/src/provider-pipeline.ts | 12 +- 51 files changed, 2654 insertions(+), 64 deletions(-) create mode 100644 packages/provider-azure/src/stages/fill-namespace-descriptions.ts create mode 100644 packages/provider-copilot/__tests__/stages/pipelines_test.ts create mode 100644 packages/provider-copilot/src/chat-facts.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/apply-top-level-cache-control.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/handle-speed-fast.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/normalize-anthropic-beta.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/promote-thinking-display.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/set-compact-headers.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/strip-cache-control-extensions.ts create mode 100644 packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts create mode 100644 packages/provider-copilot/src/stages/authenticate-chat.ts create mode 100644 packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts create mode 100644 packages/provider-copilot/src/stages/openai-chat-completions/attach-cache-control-markers.ts create mode 100644 packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts create mode 100644 packages/provider-copilot/src/stages/openai-chat-completions/set-initiator-header.ts create mode 100644 packages/provider-copilot/src/stages/openai-chat-completions/set-vision-header.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/compress-images.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/fill-empty-namespace-descriptions.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/force-store-false.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/set-initiator-header.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/set-vision-header.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/strip-image-generation.ts create mode 100644 packages/provider-copilot/src/stages/openai-responses/strip-service-tier.ts create mode 100644 packages/provider-copilot/src/stages/prepare-chat-request.ts create mode 100644 packages/provider-copilot/src/stages/reshape-compaction.ts create mode 100644 packages/provider-copilot/src/stages/select-chat-model.ts create mode 100644 packages/provider-custom/__tests__/stages/pipelines_test.ts create mode 100644 packages/provider/__tests__/stages/decode-response_test.ts create mode 100644 packages/provider/src/stages/decode-response.ts create mode 100644 packages/provider/src/stages/observe-call.ts create mode 100644 packages/provider/src/stages/select-responses-action.ts diff --git a/packages/provider-azure/src/pipelines.ts b/packages/provider-azure/src/pipelines.ts index 61803e73d2..776a824d64 100644 --- a/packages/provider-azure/src/pipelines.ts +++ b/packages/provider-azure/src/pipelines.ts @@ -1,13 +1,20 @@ import type { AzureUpstreamConfig } from './config.ts'; +import { fillAzureNamespaceDescriptions } from './stages/fill-namespace-descriptions.ts'; import { prepareAzureRequest } from './stages/prepare-request.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createAzurePipelines = (config: AzureUpstreamConfig): ProviderPipelines => ({ - openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('azure.openaiCompletions', [prepareAzureRequest(config, 'openaiCompletions'), http]), - openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('azure.openaiEmbeddings', [prepareAzureRequest(config, 'openaiEmbeddings'), http]), - openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('azure.openaiImagesGenerations', [prepareAzureRequest(config, 'openaiImagesGenerations'), http]), - openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('azure.openaiImagesEdits', [prepareAzureRequest(config, 'openaiImagesEdits'), http]), - openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('azure.openaiAudioTranscriptions', [prepareAzureRequest(config, 'openaiAudioTranscriptions'), http]), + openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('azure.openaiChatCompletions', [decodeProviderResponse('openaiChatCompletions'), prepareAzureRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('azure.openaiResponses', [selectProviderResponsesAction('generate'), fillAzureNamespaceDescriptions, decodeProviderResponse('openaiResponses'), prepareAzureRequest(config, 'openaiResponses'), observeProviderCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('azure.openaiResponsesCompact', [selectProviderResponsesAction('compact'), fillAzureNamespaceDescriptions, decodeProviderResponse('openaiResponsesCompact'), prepareAzureRequest(config, 'openaiResponsesCompact'), observeProviderCall, http]), + anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('azure.anthropicMessages', [decodeProviderResponse('anthropicMessages'), prepareAzureRequest(config, 'anthropicMessages'), observeProviderCall, http]), + anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('azure.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareAzureRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('azure.openaiCompletions', [prepareAzureRequest(config, 'openaiCompletions'), observeProviderCall, http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('azure.openaiEmbeddings', [prepareAzureRequest(config, 'openaiEmbeddings'), observeProviderCall, http]), + openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('azure.openaiImagesGenerations', [prepareAzureRequest(config, 'openaiImagesGenerations'), observeProviderCall, http]), + openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('azure.openaiImagesEdits', [prepareAzureRequest(config, 'openaiImagesEdits'), observeProviderCall, http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('azure.openaiAudioTranscriptions', [prepareAzureRequest(config, 'openaiAudioTranscriptions'), observeProviderCall, http]), }); diff --git a/packages/provider-azure/src/stages/fill-namespace-descriptions.ts b/packages/provider-azure/src/stages/fill-namespace-descriptions.ts new file mode 100644 index 0000000000..7a8387c7ce --- /dev/null +++ b/packages/provider-azure/src/stages/fill-namespace-descriptions.ts @@ -0,0 +1,21 @@ +import { defineStage, move } from '@floway-dev/pipeline'; +import { mapOpenAIResponsesTools, type CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ProviderModelFacts } from '@floway-dev/provider'; + +type Facts = { 'request.provider.payload': Omit; 'request.provider.model': ProviderModelFacts }; + +// Azure retains the former namespace-description minimum length. +// https://github.com/openai/openai-openapi/commit/466c74a42f51c02f1927bc666815251dc53845dc +// https://github.com/openai/codex/issues/37380 +export const fillAzureNamespaceDescriptions = defineStage({ + name: 'fillAzureNamespaceDescriptions', + through: { + request: { needs: ['request.provider.payload', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const { model: _, ...payload } = mapOpenAIResponsesTools({ ...facts['request.provider.payload'], model: facts['request.provider.model'].id }, tool => tool.type === 'namespace' && tool.description === '' + ? { ...tool, description: `Tools in the ${tool.name} namespace.` } : tool); + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + }, +}); diff --git a/packages/provider-azure/src/stages/prepare-request.ts b/packages/provider-azure/src/stages/prepare-request.ts index b0b1ed9ae4..7d8fa3f753 100644 --- a/packages/provider-azure/src/stages/prepare-request.ts +++ b/packages/provider-azure/src/stages/prepare-request.ts @@ -1,17 +1,24 @@ import type { AzureUpstreamConfig } from '../config.ts'; import { azureOpenAiV1BaseUrl } from '../endpoint.ts'; -import { type HttpRequestFacts, type HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { type HttpRequestFacts } from '@floway-dev/http/pipeline'; import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { joinBaseAndPath, mergeHttpHeaders, prepareOpenAIImagesEditsBody, withHttpContentType, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; -type AzureOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiAudioTranscriptions'; +type AzureOperation = 'openaiChatCompletions' | 'openaiResponses' | 'openaiResponsesCompact' | 'anthropicMessages' | 'anthropicMessagesCountTokens' | 'openaiCompletions' | 'openaiEmbeddings' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiAudioTranscriptions'; interface AzureHttpFacts extends HttpRequestFacts { + 'request.provider.modelKey': string; 'request.azure.deployment': string; } const PATHS: Record, string> = { + openaiChatCompletions: '/chat/completions', + openaiResponses: '/responses', + openaiResponsesCompact: '/responses/compact', + anthropicMessages: '/messages', + anthropicMessagesCountTokens: '/messages/count_tokens', openaiCompletions: '/completions', openaiEmbeddings: '/embeddings', openaiImagesGenerations: '/images/generations', @@ -19,15 +26,15 @@ const PATHS: Record, string }; export const prepareAzureRequest = (config: AzureUpstreamConfig, operation: O) => { - const prepare = defineStage, AzureHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ + const prepare = defineStage & { 'request.provider.anthropicBeta'?: readonly string[]; 'request.provider.responsesAction': 'generate' | 'compact' }, AzureHttpFacts, ProviderResponse, ProviderResponse, ProviderServices>({ name: `prepareAzure${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, through: { request: { - needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + needs: [...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : [])], consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.azure.deployment'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.provider.modelKey', 'request.azure.deployment'], }, - response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next) => { const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; @@ -45,20 +52,28 @@ export const prepareAzureRequest = (config: AzureUpstr // https://github.com/Azure/azure-rest-api-specs/blob/928047803788f7377fa003a26ba2bdc2e0fcccc0/specification/cognitiveservices/OpenAI.Inference/routes/audio_transcription.tsp#L19-L49 url.searchParams.set('api-version', '2025-04-01-preview'); } else { - url = new URL(joinBaseAndPath(azureOpenAiV1BaseUrl(config.endpoint), PATHS[operation as Exclude])); + url = new URL(joinBaseAndPath(azureOpenAiV1BaseUrl(config.endpoint), ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') ? PATHS[facts['request.provider.responsesAction'] === 'compact' ? 'openaiResponsesCompact' : 'openaiResponses'] : PATHS[operation as Exclude]))); if (operation === 'openaiImagesEdits') { const prepared = await prepareOpenAIImagesEditsBody(payload as ProviderOperationPayloads['openaiImagesEdits'], deployment); body = prepared.body; encoding = prepared.encoding; } else { - body = { ...payload, model: deployment }; + body = (operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact' + ? { ...toCompactPayloadShape(payload as ProviderOperationPayloads['openaiResponsesCompact']), model: deployment } + : { ...payload, ...(operation === 'openaiChatCompletions' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' || operation === 'anthropicMessages' ? { stream: true } : {}), model: deployment }; } if (operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') url.searchParams.append('api-version', 'preview'); } const base = withHttpContentType([['api-key', secret(config.apiKey)]], body, encoding); - const headers = mergeHttpHeaders(base, facts['request.http.headers']); + let headers = mergeHttpHeaders(base, facts['request.http.headers']); + if ('request.provider.anthropicBeta' in facts) { + const beta = facts['request.provider.anthropicBeta'] as readonly string[]; + headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); + if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; + } const back = await next(move({ ...rest, + 'request.provider.modelKey': deployment, 'request.azure.deployment': deployment, 'request.http.url': url.href, 'request.http.method': 'POST', @@ -66,7 +81,7 @@ export const prepareAzureRequest = (config: AzureUpstr 'request.http.body': body, 'request.http.encoding': encoding, })); - return move({ ...back, 'response.provider.modelKey': deployment, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + return move({ ...back }); }, }); return prepare; diff --git a/packages/provider-copilot/__tests__/stages/pipelines_test.ts b/packages/provider-copilot/__tests__/stages/pipelines_test.ts new file mode 100644 index 0000000000..7d7f6867f4 --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/pipelines_test.ts @@ -0,0 +1,70 @@ +import { expect, test } from 'vitest'; + +import { clearInProcessCopilotTokenCache } from '../../src/auth.ts'; +import { createCopilotPipelines } from '../../src/pipelines.ts'; +import { emptyCopilotUpstreamState } from '../../src/state.ts'; +import { move, run } from '@floway-dev/pipeline'; +import { initProviderRepo, providerModelFacts, type ProviderChatServices, type ProviderOperationRequest, type UpstreamRecord } from '@floway-dev/provider'; +import { jsonResponse, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; + +const auth = { id: 'up_pipeline', githubHost: 'github.com', githubToken: 'test-token' }; +const setup = (): void => { + clearInProcessCopilotTokenCache(); + const upstream: UpstreamRecord = { id: auth.id, kind: 'copilot', name: 'Test', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: {}, state: { ...emptyCopilotUpstreamState(), copilotToken: { token: 'access-token', expiresAt: 4102444800, baseUrl: 'https://copilot.example' } }, flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210 }; + initProviderRepo(() => ({ upstreams: { getById: async () => upstream, saveState: async () => {} } })); +}; +const services: ProviderChatServices = { httpCall: () => ({ ...noopUpstreamCallOptions(), signal: undefined }), recordProtocolFrames: frames => frames }; + +test('Copilot compaction runs vendor request stages and normalizes the generated value in the same run', async () => { + setup(); + const pipeline = createCopilotPipelines(auth).openaiResponsesCompact; + if (pipeline === undefined) throw new Error('Missing compact chain'); + const model = stubProviderModel({ id: 'gpt-5.4', providerData: { rawModels: [{ id: 'gpt-5.4', supported_endpoints: ['/responses'] }] } }); + const input: ProviderOperationRequest<'openaiResponsesCompact'> = move({ + 'request.provider.model': providerModelFacts(model), + 'request.provider.payload': { input: [{ type: 'message', role: 'user', content: 'hello' }], store: true, service_tier: 'priority', tools: [{ type: 'namespace', name: 'tools', description: '', tools: [] }] }, + 'request.http.callId': 1, + 'request.http.headers': [], + }); + await withMockedFetch(async request => { + expect(request.url).toBe('https://copilot.example/responses'); + expect(request.headers.get('authorization')).toBe('Bearer access-token'); + const body = await request.json() as Record; + expect(body).toMatchObject({ model: 'gpt-5.4', store: false, stream: false, tools: [{ type: 'namespace', description: 'Tools in the tools namespace.' }] }); + expect(body).not.toHaveProperty('service_tier'); + expect(body.input).toEqual([{ type: 'message', role: 'user', content: 'hello' }, { type: 'compaction_trigger' }]); + return jsonResponse({ id: 'response', object: 'response', output: [{ type: 'compaction', id: 'upstream-compaction-id', encrypted_content: 'opaque' }] }); + }, async () => { + const executed = await run(pipeline, input, services); + const output = executed.facts['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'value') throw new Error('Expected compact value'); + expect(output.body.object).toBe('response.compaction'); + expect(output.body.output).toHaveLength(2); + expect(output.body.output[1]).toMatchObject({ type: 'compaction' }); + expect(output.body.output[1].id).not.toBe('upstream-compaction-id'); + expect(executed.facts['response.provider.called']).toBe(true); + expect(executed.facts['response.provider.modelKey']).toBe('gpt-5.4'); + expect(input['request.provider.payload'].store).toBe(true); + await executed.drain(); + }); +}); + +test('unsupported Anthropic Fast Mode is a preflight value with no model endpoint observation', async () => { + setup(); + const pipeline = createCopilotPipelines(auth).anthropicMessages; + if (pipeline === undefined) throw new Error('Missing Messages chain'); + const input: ProviderOperationRequest<'anthropicMessages'> = move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ id: 'claude-sonnet-4.6', providerData: { rawModels: [{ id: 'claude-sonnet-4.6', supported_endpoints: ['/v1/messages'] }] } })), + 'request.provider.payload': { max_tokens: 16, messages: [], speed: 'fast' }, + 'request.provider.anthropicBeta': [], + 'request.http.callId': 1, + 'request.http.headers': [], + }); + await withMockedFetch(() => { throw new Error('No network request expected'); }, async () => { + const executed = await run(pipeline, input, services); + expect(executed.facts['response.provider.called']).toBe(false); + expect(executed.facts['response.provider.previousCalls']).toEqual([]); + expect(executed.facts['response.http.exchange']).toMatchObject({ type: 'response', status: 400 }); + await executed.drain(); + }); +}); diff --git a/packages/provider-copilot/src/chat-facts.ts b/packages/provider-copilot/src/chat-facts.ts new file mode 100644 index 0000000000..e8c711b416 --- /dev/null +++ b/packages/provider-copilot/src/chat-facts.ts @@ -0,0 +1,22 @@ +import type { CopilotSession } from './pipeline-facts.ts'; +import type { HttpHeaders, HttpRequestFacts } from '@floway-dev/http/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ProviderModelFacts } from '@floway-dev/provider'; + +export interface ChatFacts

{ + 'request.provider.model': ProviderModelFacts; + 'request.provider.payload': P; + 'request.provider.modelKey': string; + 'request.http.callId': number; + 'request.http.headers': HttpHeaders; +} +export type ChatCompletionsFacts = ChatFacts; +export type ResponsesFacts = ChatFacts & { 'request.provider.responsesAction': 'generate' | 'compact' }; +export type MessagesFacts = ChatFacts & { 'request.provider.anthropicBeta': readonly string[] }; +export type CopilotChatFacts = ChatFacts & { 'request.provider.anthropicBeta'?: readonly string[]; 'request.provider.responsesAction'?: 'generate' | 'compact' }; +export type AuthenticatedChatFacts = CopilotChatFacts & { 'request.copilot.session': CopilotSession }; +export interface CopilotChatHttpFacts extends HttpRequestFacts { + 'request.provider.modelKey': string; +} diff --git a/packages/provider-copilot/src/pipeline-facts.ts b/packages/provider-copilot/src/pipeline-facts.ts index cf269f3767..1029fe7b43 100644 --- a/packages/provider-copilot/src/pipeline-facts.ts +++ b/packages/provider-copilot/src/pipeline-facts.ts @@ -13,5 +13,6 @@ export interface CopilotRequest extends EmbeddingsRequest { } export interface CopilotHttpFacts extends HttpRequestFacts { 'request.copilot.modelKey': string; + 'request.provider.modelKey': string; } export type QuotaResponse = HttpResponseFacts & { 'response.copilot.quota': Deferred }; diff --git a/packages/provider-copilot/src/pipelines.ts b/packages/provider-copilot/src/pipelines.ts index 73636dc2d8..1fd7c3e642 100644 --- a/packages/provider-copilot/src/pipelines.ts +++ b/packages/provider-copilot/src/pipelines.ts @@ -1,12 +1,48 @@ import type { CopilotAuth } from './auth.ts'; import type { EmbeddingsRequest } from './pipeline-facts.ts'; +import { copilotAnthropicMessagesApplyTopLevelCacheControl } from './stages/anthropic-messages/apply-top-level-cache-control.ts'; +import { copilotAnthropicMessagesCompressImages } from './stages/anthropic-messages/compress-images.ts'; +import { copilotAnthropicMessagesSpeedFast } from './stages/anthropic-messages/handle-speed-fast.ts'; +import { copilotAnthropicMessagesNormalizeAnthropicBeta } from './stages/anthropic-messages/normalize-anthropic-beta.ts'; +import { copilotAnthropicMessagesThinkingDisplay } from './stages/anthropic-messages/promote-thinking-display.ts'; +import { rewriteCopilotContextWindowError } from './stages/anthropic-messages/rewrite-context-window-error.ts'; +import { copilotAnthropicMessagesSetClaudeAgentHeaders } from './stages/anthropic-messages/set-claude-agent-headers.ts'; +import { copilotAnthropicMessagesSetCompactHeaders } from './stages/anthropic-messages/set-compact-headers.ts'; +import { copilotAnthropicMessagesSetInitiatorHeader } from './stages/anthropic-messages/set-initiator-header.ts'; +import { copilotAnthropicMessagesSetInteractionIdHeader } from './stages/anthropic-messages/set-interaction-id-header.ts'; +import { copilotAnthropicMessagesSetVisionHeader } from './stages/anthropic-messages/set-vision-header.ts'; +import { copilotAnthropicMessagesStripCacheControlExtensions } from './stages/anthropic-messages/strip-cache-control-extensions.ts'; +import { copilotAnthropicMessagesStripEagerInputStreaming } from './stages/anthropic-messages/strip-eager-input-streaming.ts'; +import { authenticateCopilotChat } from './stages/authenticate-chat.ts'; import { authenticateCopilot } from './stages/authenticate.ts'; import { observeCopilotQuota } from './stages/observe-quota.ts'; +import { copilotOpenAIChatCompletionsAbortToolWhitespace } from './stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts'; +import { copilotOpenAIChatCompletionsAttachCacheControlMarkers } from './stages/openai-chat-completions/attach-cache-control-markers.ts'; +import { copilotOpenAIChatCompletionsCompressImages } from './stages/openai-chat-completions/compress-images.ts'; +import { copilotOpenAIChatCompletionsSetInitiatorHeader } from './stages/openai-chat-completions/set-initiator-header.ts'; +import { copilotOpenAIChatCompletionsSetVisionHeader } from './stages/openai-chat-completions/set-vision-header.ts'; +import { copilotOpenAIResponsesAbortToolWhitespace } from './stages/openai-responses/abort-on-tool-argument-whitespace.ts'; +import { copilotOpenAIResponsesCompressImages } from './stages/openai-responses/compress-images.ts'; +import { copilotOpenAIResponsesFillEmptyNamespaceDescriptions } from './stages/openai-responses/fill-empty-namespace-descriptions.ts'; +import { copilotOpenAIResponsesForceStoreFalse } from './stages/openai-responses/force-store-false.ts'; +import { copilotOpenAIResponsesItemIdMembrane } from './stages/openai-responses/item-id-membrane.ts'; +import { copilotOpenAIResponsesSetInitiatorHeader } from './stages/openai-responses/set-initiator-header.ts'; +import { copilotOpenAIResponsesSetVisionHeader } from './stages/openai-responses/set-vision-header.ts'; +import { copilotOpenAIResponsesStripImageGeneration } from './stages/openai-responses/strip-image-generation.ts'; +import { copilotOpenAIResponsesStripServiceTier } from './stages/openai-responses/strip-service-tier.ts'; +import { prepareCopilotChatRequest } from './stages/prepare-chat-request.ts'; import { prepareCopilotEmbeddings } from './stages/prepare-request.ts'; +import { reshapeCopilotCompaction } from './stages/reshape-compaction.ts'; +import { selectCopilotChatModel } from './stages/select-chat-model.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderPipelines, ProviderResponse } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, type ProviderOperationRequest, type ProviderOperationResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; export const createCopilotPipelines = (auth: CopilotAuth): ProviderPipelines => ({ - openaiEmbeddings: compose('copilot.openaiEmbeddings', [authenticateCopilot(auth), prepareCopilotEmbeddings(), observeCopilotQuota(auth.id), http]), + openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('copilot.openaiChatCompletions', [selectCopilotChatModel('openaiChatCompletions'), authenticateCopilotChat(auth), copilotOpenAIChatCompletionsCompressImages, copilotOpenAIChatCompletionsAbortToolWhitespace, copilotOpenAIChatCompletionsAttachCacheControlMarkers, copilotOpenAIChatCompletionsSetInitiatorHeader, copilotOpenAIChatCompletionsSetVisionHeader, decodeProviderResponse('openaiChatCompletions'), prepareCopilotChatRequest('openaiChatCompletions'), observeCopilotQuota(auth.id), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('copilot.openaiResponses', [selectCopilotChatModel('openaiResponses'), authenticateCopilotChat(auth), copilotOpenAIResponsesCompressImages, copilotOpenAIResponsesFillEmptyNamespaceDescriptions, copilotOpenAIResponsesStripServiceTier, copilotOpenAIResponsesStripImageGeneration, copilotOpenAIResponsesForceStoreFalse, copilotOpenAIResponsesItemIdMembrane, copilotOpenAIResponsesAbortToolWhitespace, copilotOpenAIResponsesSetVisionHeader, copilotOpenAIResponsesSetInitiatorHeader, decodeProviderResponse('openaiResponses'), prepareCopilotChatRequest('openaiResponses'), observeCopilotQuota(auth.id), observeProviderCall, http]), + anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('copilot.anthropicMessages', [selectCopilotChatModel('anthropicMessages'), authenticateCopilotChat(auth), rewriteCopilotContextWindowError, copilotAnthropicMessagesSetCompactHeaders, copilotAnthropicMessagesSetClaudeAgentHeaders, copilotAnthropicMessagesSetInteractionIdHeader, copilotAnthropicMessagesCompressImages, copilotAnthropicMessagesSpeedFast, copilotAnthropicMessagesThinkingDisplay, copilotAnthropicMessagesApplyTopLevelCacheControl, copilotAnthropicMessagesStripCacheControlExtensions, copilotAnthropicMessagesStripEagerInputStreaming, copilotAnthropicMessagesSetVisionHeader, copilotAnthropicMessagesSetInitiatorHeader, copilotAnthropicMessagesNormalizeAnthropicBeta, decodeProviderResponse('anthropicMessages'), prepareCopilotChatRequest('anthropicMessages'), observeCopilotQuota(auth.id), observeProviderCall, http]), + anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('copilot.anthropicMessagesCountTokens', [selectCopilotChatModel('anthropicMessagesCountTokens'), authenticateCopilotChat(auth), copilotAnthropicMessagesCompressImages, copilotAnthropicMessagesSetVisionHeader, copilotAnthropicMessagesSetInitiatorHeader, copilotAnthropicMessagesNormalizeAnthropicBeta, decodeProviderResponse('anthropicMessagesCountTokens'), prepareCopilotChatRequest('anthropicMessagesCountTokens'), observeCopilotQuota(auth.id), observeProviderCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('copilot.openaiResponsesCompact', [selectCopilotChatModel('openaiResponsesCompact'), authenticateCopilotChat(auth), copilotOpenAIResponsesCompressImages, copilotOpenAIResponsesFillEmptyNamespaceDescriptions, copilotOpenAIResponsesStripServiceTier, copilotOpenAIResponsesStripImageGeneration, copilotOpenAIResponsesForceStoreFalse, copilotOpenAIResponsesItemIdMembrane, copilotOpenAIResponsesAbortToolWhitespace, copilotOpenAIResponsesSetVisionHeader, copilotOpenAIResponsesSetInitiatorHeader, reshapeCopilotCompaction, decodeProviderResponse('openaiResponsesCompact'), prepareCopilotChatRequest('openaiResponsesCompact'), observeCopilotQuota(auth.id), observeProviderCall, http]), + openaiEmbeddings: compose('copilot.openaiEmbeddings', [authenticateCopilot(auth), prepareCopilotEmbeddings(), observeCopilotQuota(auth.id), observeProviderCall, http]), }); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/apply-top-level-cache-control.ts b/packages/provider-copilot/src/stages/anthropic-messages/apply-top-level-cache-control.ts new file mode 100644 index 0000000000..0920d907bf --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/apply-top-level-cache-control.ts @@ -0,0 +1,96 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { + AnthropicMessagesAssistantInputContentBlock, + AnthropicMessagesPayload, + AnthropicMessagesTextBlock, + AnthropicMessagesUserContentBlock, +} from '@floway-dev/protocols/anthropic-messages'; + +/** + * Anthropic's Messages API defines a top-level `cache_control` field that + * "automatically applies a cache_control marker to the last cacheable block + * in the request" (per `MessageCreateParamsBase` in the official SDK). + * Copilot's `/v1/messages` deployment validates against an older schema for + * several model slots (claude-haiku-4.5, claude-sonnet-4.5, claude-sonnet-4.6, + * and intermittently claude-opus-4.5) and rejects the top-level field with + * `cache_control: Extra inputs are not permitted`. Newer slots (opus 4.6/4.7) + * silently accept it. + * + * Port the marker onto the last cacheable content block (mirroring the + * documented semantics), then drop the top-level field. If the last cacheable + * block already carries its own `cache_control`, leave it alone — an explicit + * marker wins over the auto-apply. Sub-field extensions (`scope`, `ttl`) + * carried in the ported value are cleaned up by + * `withCacheControlExtensionsStripped`, which runs immediately after. + * + * References: + * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L2909-L2913 + * - https://github.com/caozhiyuan/copilot-api/issues/269 + */ + +type CacheableBlock = Extract< + AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock, + { cache_control?: unknown } +>; + +const isCacheableBlock = (block: AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock): block is CacheableBlock => + block.type === 'text' || block.type === 'image' || block.type === 'tool_use' || block.type === 'tool_result'; + +/** Where the marker lands: the last message that can carry one, and within it either the whole + * string content or the last cacheable block. Reading the position out first is what lets the + * rewrite below rebuild one path and leave every other message alone. */ +const markerSite = (messages: AnthropicMessagesPayload['messages']): { readonly message: number; readonly block: number | 'whole' } | undefined => { + for (let m = messages.length - 1; m >= 0; m--) { + const content = messages[m].content; + if (typeof content === 'string') return { message: m, block: 'whole' }; + for (let b = content.length - 1; b >= 0; b--) { + if (isCacheableBlock(content[b])) return { message: m, block: b }; + } + } + return undefined; +}; + +export const copilotAnthropicMessagesApplyTopLevelCacheControl = defineStage({ + name: 'copilotAnthropicMessagesApplyTopLevelCacheControl', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload'] as typeof facts['request.provider.payload'] & { cache_control?: { type: 'ephemeral' } }; + + const topLevel = payload.cache_control; + if (topLevel === undefined) return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + const { cache_control: _ported, ...withoutTopLevel } = payload; + const site = markerSite(payload.messages); + if (site === undefined) { + payload = withoutTopLevel; + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + } + + payload = { + ...withoutTopLevel, + // Each rewrite keeps its message's own role and only re-homes that role's own blocks, so + // the array is handed back at its element type; spreading alone widens it to the union. + messages: payload.messages.map((message, m): AnthropicMessagesPayload['messages'][number] => { + if (m !== site.message) return message; + if (site.block === 'whole') { + const block: AnthropicMessagesTextBlock = { type: 'text', text: message.content as string, cache_control: topLevel }; + return { ...message, content: [block] } as AnthropicMessagesPayload['messages'][number]; + } + const content = message.content as readonly CacheableBlock[]; + // `??=` semantics: a block that already states its own marker keeps it. + if (content[site.block].cache_control !== undefined) return message; + return { + ...message, + content: content.map((block, b) => (b === site.block ? { ...block, cache_control: topLevel } : block)), + } as AnthropicMessagesPayload['messages'][number]; + }), + }; + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts b/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts new file mode 100644 index 0000000000..9e48309b0a --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts @@ -0,0 +1,123 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { memoizedBase64Compressor } from '../../interceptors/image-compression.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { type ImageSizeCalculator, type SizeCaps, fitWithin } from '@floway-dev/platform'; +import type { AnthropicMessagesPayload, AnthropicMessagesImageBlock, AnthropicMessagesMessage, AnthropicMessagesToolResultBlock, AnthropicMessagesToolResultContentBlock, AnthropicMessagesUserContentBlock } from '@floway-dev/protocols/anthropic-messages'; +import { type ProviderModelFacts } from '@floway-dev/provider'; + +// Per-model image caps for the Claude (Anthropic Messages) egress, measured from the real +// /v1/messages generation path (count_tokens misreports the downscale here). +// Copilot's Anthropic Messages endpoint is Anthropic-only by Copilot's own contract, so +// the caps below are Anthropic's canonical vision limits: +// +// - Opus 4.7 was the first high-resolution Claude model; Opus 4.7 and 4.8 +// sample up to ~3.59 MP within a 2576px long edge. +// - Opus 4.5 / 4.6 and the sonnet / haiku families use the standard ~1.18 MP / +// 1568px cap. +// +// We threshold on the Opus version so future Opus releases inherit the high-res +// tier; every non-Opus Claude model uses the standard cap. +// +// References: +// - Anthropic vision docs: https://docs.claude.com/en/docs/build-with-claude/vision +// - High-res Opus 4.7+ sampling: https://docs.claude.com/en/docs/build-with-claude/vision#evaluate-image-size +const STANDARD_CLAUDE_CAPS: SizeCaps = { maxLongEdge: 1568, maxArea: 1_176_000 }; + +const claudeImageCaps = (upstreamModelId: string): SizeCaps => { + const opus = /opus-(\d+)(?:\.(\d+))?/.exec(upstreamModelId); + if (!opus) return STANDARD_CLAUDE_CAPS; + const major = Number(opus[1]); + const minor = opus[2] === undefined ? 0 : Number(opus[2]); + const highRes = major > 4 || (major === 4 && minor >= 7); + return highRes ? { maxLongEdge: 2576, maxArea: 3_588_000 } : STANDARD_CLAUDE_CAPS; +}; + +// Anthropic carries inline images both at the top level of a message's content +// and nested inside tool_result content, mirroring the vision-header scan. +const collectImageBlocks = (messages: AnthropicMessagesMessage[]): AnthropicMessagesImageBlock[] => { + const blocks: AnthropicMessagesImageBlock[] = []; + for (const message of messages) { + if (!Array.isArray(message.content)) continue; + for (const block of message.content) { + if (block.type === 'image') blocks.push(block); + else if (block.type === 'tool_result' && Array.isArray(block.content)) { + for (const inner of block.content) { + if (inner.type === 'image') blocks.push(inner); + } + } + } + } + return blocks; +}; + +const compressInlineImages = async (payload: AnthropicMessagesPayload, model: ProviderModelFacts): Promise => { + const blocks = collectImageBlocks(payload.messages); + if (blocks.length === 0) return payload; + + const caps = claudeImageCaps(model.id); + const targetSize: ImageSizeCalculator = source => fitWithin(source, caps); + const compress = memoizedBase64Compressor(targetSize); + const compressedData = new Map(); + await Promise.all( + blocks.map(async block => { + compressedData.set(block, await compress(block.source.data)); + }), + ); + const hasCompressedImage = (block: AnthropicMessagesToolResultContentBlock | AnthropicMessagesUserContentBlock): block is AnthropicMessagesImageBlock => + block.type === 'image' && compressedData.has(block); + const hasCompressedToolResultImage = ( + block: AnthropicMessagesUserContentBlock, + ): block is AnthropicMessagesToolResultBlock & { content: AnthropicMessagesToolResultContentBlock[] } => + block.type === 'tool_result' + && Array.isArray(block.content) + && block.content.some(hasCompressedImage); + const rewriteImage = (block: AnthropicMessagesImageBlock): AnthropicMessagesImageBlock => { + const data = compressedData.get(block); + if (data === undefined) throw new Error('Missing compressed Anthropic Messages image data'); + return { + ...block, + source: { ...block.source, media_type: 'image/webp', data }, + }; + }; + const rewriteUserContent = (content: AnthropicMessagesUserContentBlock[]): AnthropicMessagesUserContentBlock[] => + content.map(block => { + if (hasCompressedImage(block)) return rewriteImage(block); + if (!hasCompressedToolResultImage(block)) return block; + return { + ...block, + content: block.content.map(inner => hasCompressedImage(inner) ? rewriteImage(inner) : inner), + }; + }); + + return { + ...payload, + messages: payload.messages.map(message => + message.role === 'user' + && Array.isArray(message.content) + && message.content.some(block => hasCompressedImage(block) || hasCompressedToolResultImage(block)) + ? { ...message, content: rewriteUserContent(message.content) } + : message), + }; +}; + +// Recompresses every inline base64 image in the outgoing Anthropic Messages payload to +// WebP before the Copilot upstream call. Generic in the run-result type so the +// same definition serves both the streaming Anthropic Messages boundary chain and the +// count_tokens boundary chain, so count_tokens sizes the same recompressed +// payload the chat path sends. + +export const copilotAnthropicMessagesCompressImages = defineStage({ + name: 'copilotAnthropicMessagesCompressImages', + through: { + request: { needs: ['request.provider.payload', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + const model = facts['request.provider.model']; + + payload = await compressInlineImages(payload, model); + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/handle-speed-fast.ts b/packages/provider-copilot/src/stages/anthropic-messages/handle-speed-fast.ts new file mode 100644 index 0000000000..f89892f5a1 --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/handle-speed-fast.ts @@ -0,0 +1,76 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +/** + * Anthropic Fast Mode is a per-request opt-in carried by `speed: "fast"` on + * the Anthropic Messages request and echoed back as `usage.speed: "fast"`. Copilot + * does not speak Fast Mode on the wire — the upstream resolves the tier via + * the `-fast` raw model id picked by `model-selection.ts` and never echoes + * `usage.speed`. This stage bridges the two contracts at the Copilot + * boundary: + * + * - Strip `speed: 'fast' | 'standard'` from the outbound payload. The + * value was already consumed by model selection to pick the raw variant + * and pre-validate Fast Mode support, so passing it through to Copilot + * would just trigger an unknown-field 400 from a strict upstream. + * - Leave any other `speed` value untouched. Unknown values mean the + * caller is wrong; let Copilot surface the same invalid_request_error + * Anthropic itself would, rather than the gateway lying about which + * field the upstream rejected. + * - When the caller asked for Fast Mode, stamp `usage.speed = 'fast'` + * onto every `message_start` and `message_delta` frame on the way out + * so downstream sees the marker the billing path (`speed` → tier='fast' + * → the `serviceTier: 'fast'` pricing entry) and Anthropic-compatible clients expect. + * + * References: + * - https://docs.claude.com/en/build-with-claude/fast-mode + * - https://docs.claude.com/en/api/service-tiers + */ + +const stampFastSpeedOntoUsage = async function* ( + frames: AsyncIterable>, +): AsyncGenerator> { + for await (const frame of frames) { + if (frame.type === 'done') { + yield frame; + continue; + } + const { event } = frame; + if (event.type === 'message_start') { + yield eventFrame({ + ...event, + message: { + ...event.message, + usage: { ...event.message.usage, speed: 'fast' }, + }, + }); + continue; + } + if (event.type === 'message_delta' && event.usage) { + yield eventFrame({ + ...event, + usage: { ...event.usage, speed: 'fast' }, + }); + continue; + } + yield frame; + } +}; + +export const copilotAnthropicMessagesSpeedFast = defineStage, ProviderChatResponse<'anthropicMessages'>, ProviderChatServices>({ + name: 'copilotAnthropicMessagesSpeedFast', + through: { request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] } }, + execute: async (facts, next, use) => { + let payload = facts['request.provider.payload']; + const speed = payload.speed; + const stamp = speed === 'fast'; + if (speed === 'fast' || speed === 'standard') { const { speed: _, ...withoutSpeed } = payload; payload = withoutSpeed; } + const back = await next(move({ ...facts, 'request.provider.payload': payload })); + const output = back['response.provider.output']; + if (!stamp || output === null || !('kind' in output) || output.kind !== 'stream') return move({ ...back }); + return move({ ...back, 'response.provider.output': { ...output, frames: use.recordProtocolFrames(stampFastSpeedOntoUsage(output.frames)) } }); + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/normalize-anthropic-beta.ts b/packages/provider-copilot/src/stages/anthropic-messages/normalize-anthropic-beta.ts new file mode 100644 index 0000000000..5224be068e --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/normalize-anthropic-beta.ts @@ -0,0 +1,44 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +const ALLOWED_ANTHROPIC_BETAS = new Set([ + 'interleaved-thinking-2025-05-14', + 'context-management-2025-06-27', + 'advanced-tool-use-2025-11-20', +]); +const INTERLEAVED_THINKING_BETA = 'interleaved-thinking-2025-05-14'; +const CONTEXT_MANAGEMENT_BETA = 'context-management-2025-06-27'; + +// Copilot rejects unknown beta values. Preserve the supported subset of the +// caller's Anthropic Messages beta intent, synthesize VS Code's thinking default only +// when the caller supplied no beta intent, and keep context_management paired +// with its required token. +// https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/platform/endpoint/node/chatEndpoint.ts#L262-L282 +// https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/extension/chatSessions/claude/node/claudeLanguageModelServer.ts#L413-L427 + +export const copilotAnthropicMessagesNormalizeAnthropicBeta = defineStage({ + name: 'copilotAnthropicMessagesNormalizeAnthropicBeta', + through: { + request: { needs: ['request.provider.payload', 'request.provider.anthropicBeta'], consumes: ['request.provider.anthropicBeta'], provides: ['request.provider.anthropicBeta'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload'] as typeof facts['request.provider.payload'] & { context_management?: unknown }; + let beta = facts['request.provider.anthropicBeta']; + + const callerSuppliedBeta = beta.length > 0; + beta = [...new Set(beta.filter(beta => ALLOWED_ANTHROPIC_BETAS.has(beta)))]; + + const isAdaptiveThinking = payload.thinking?.type === 'adaptive'; + if (!callerSuppliedBeta && payload.thinking?.budget_tokens && !isAdaptiveThinking) { + beta = [...beta, INTERLEAVED_THINKING_BETA]; + } + + if (payload.context_management !== undefined && !beta.includes(CONTEXT_MANAGEMENT_BETA)) { + beta = [...beta, CONTEXT_MANAGEMENT_BETA]; + } + + return move({ ...await next(move({ ...facts, 'request.provider.anthropicBeta': beta })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/promote-thinking-display.ts b/packages/provider-copilot/src/stages/anthropic-messages/promote-thinking-display.ts new file mode 100644 index 0000000000..ee56db9dba --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/promote-thinking-display.ts @@ -0,0 +1,112 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { copilotRawModelId } from '../../model-name.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesThinkingDisplay } from '@floway-dev/protocols/anthropic-messages'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +const CLAUDE_VERSION_PATTERN = /(?:^|-)(\d+)\.(\d+)(?=-|$)/; + +const isAnthropicMessagesThinkingDisplay = (value: unknown): value is AnthropicMessagesThinkingDisplay => value === 'omitted' || value === 'summarized' || value === 'full'; + +const isClaudeVersionAtLeast = (model: string, major: number, minor: number): boolean => { + const normalized = copilotRawModelId(model); + if (!normalized.startsWith('claude-')) return false; + + const match = normalized.match(CLAUDE_VERSION_PATTERN); + if (!match) return false; + + const modelMajor = Number(match[1]); + const modelMinor = Number(match[2]); + + return modelMajor > major || (modelMajor === major && modelMinor >= minor); +}; + +export const resolveAnthropicMessagesDownstreamThinkingDisplay = (ctx: { payload: AnthropicMessagesPayload }): AnthropicMessagesThinkingDisplay | undefined => { + const display = ctx.payload.thinking?.display; + if (display !== undefined) { + // Request JSON is not runtime-validated before boundary stages; leave + // unknown display values untouched so upstream, not this workaround, owns + // rejecting or accepting future values. + return isAnthropicMessagesThinkingDisplay(display) ? display : undefined; + } + + return isClaudeVersionAtLeast(ctx.payload.model, 4, 7) ? 'omitted' : 'summarized'; +}; + +const omitThinkingTextFromProtocolFrame = (frame: ProtocolFrame): ProtocolFrame | undefined => { + if (frame.type === 'done') return frame; + + const { event } = frame; + if (event.type === 'content_block_start' && event.content_block.type === 'thinking') { + return eventFrame({ + ...event, + content_block: { + ...event.content_block, + thinking: '', + }, + }); + } + + if (event.type === 'content_block_delta' && event.delta.type === 'thinking_delta') { + return undefined; + } + + return frame; +}; + +const omitThinkingTextFromProtocolFrames = async function* (frames: AsyncIterable>): AsyncGenerator> { + for await (const frame of frames) { + const omitted = omitThinkingTextFromProtocolFrame(frame); + if (omitted) yield omitted; + } +}; + +/** + * Workaround for Copilot/Claude turns that go silent during long extended + * thinking, then surface to clients as `API Error: Network connection lost`, + * `Stream idle timeout`, or a short no-tool-call turn. The direct Copilot + * references below document a roughly 60s HTTP response-idle boundary in + * GitHub Copilot paths; the related client reports show the same user-visible + * stall/no-output failure shape in official Copilot and Claude Code clients. + * + * The native Copilot Anthropic Messages target is the boundary where `thinking.display` + * controls whether the upstream emits token-level `thinking_delta` SSE while + * the model is reasoning. Our Copilot probes found Claude 4.7 defaults to + * omitted display, while 4.6/4.5 default to summarized; forcing summarized + * upstream keeps data flowing during thinking and avoids the idle gap. To keep + * downstream omitted semantics, this boundary stage removes only thinking + * text/deltas after the upstream attempt and preserves every `signature` byte; + * the same probes showed blank thinking text is accepted, while any signature + * tampering makes the next Anthropic Messages request fail with 400. Those probes justify + * the request/response mechanics here; the public references justify why this + * workaround exists. + * + * References: + * Direct Copilot HTTP idle-boundary reports: + * - https://github.com/ericc-ch/copilot-api/issues/223 + * - https://github.com/copilot-extensions/user-feedback/issues/2 + * Related official-client and downstream symptoms: + * - https://github.com/microsoft/vscode-copilot-release/issues/7640 + * - https://github.com/github/copilot-cli/issues/686 + * - https://github.com/github/copilot-cli/issues/1614 + * - https://github.com/anthropics/claude-code/issues/46987 + * - https://github.com/anthropics/claude-code/issues/50477 + */ + +export const copilotAnthropicMessagesThinkingDisplay = defineStage, ProviderChatResponse<'anthropicMessages'>, ProviderChatServices>({ + name: 'copilotAnthropicMessagesThinkingDisplay', + through: { request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] } }, + execute: async (facts, next, use) => { + let payload = facts['request.provider.payload']; + const display = resolveAnthropicMessagesDownstreamThinkingDisplay({ payload }); + const thinking = payload.thinking; + const active = !!thinking && thinking.type !== 'disabled'; + const stamp = active && display === 'omitted'; + if (active && display !== undefined && display !== 'full') payload = { ...payload, thinking: { ...thinking, display: 'summarized' } }; + const back = await next(move({ ...facts, 'request.provider.payload': payload })); + const output = back['response.provider.output']; + if (!stamp || output === null || !('kind' in output) || output.kind !== 'stream') return move({ ...back }); + return move({ ...back, 'response.provider.output': { ...output, frames: use.recordProtocolFrames(omitThinkingTextFromProtocolFrames(output.frames)) } }); + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts b/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts new file mode 100644 index 0000000000..82954c6a4b --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts @@ -0,0 +1,30 @@ +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease } from '@floway-dev/pipeline'; +import { buildPromptTooLongBody } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const rewriteCopilotContextWindowError = defineStage, ProviderChatResponse<'anthropicMessages'> & { 'response.copilot.errorBody': unknown }>({ + name: 'rewriteCopilotContextWindowError', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body', 'response.provider.output'], consumes: ['response.http.body', 'response.provider.output'], provides: ['response.http.body', 'response.provider.output', 'response.copilot.errorBody'] }, + }, + execute: async (facts, next) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + if (exchange.type !== 'response' || exchange.status >= 200 && exchange.status < 300) return move({ ...back, 'response.copilot.errorBody': null }); + const text = await exchangeResponse(exchange).text(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + let body: unknown; + try { body = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; body = text; } + const tooLong = text.includes('Request body is too large for model context window') || text.includes('context_length_exceeded'); + const rewritten = tooLong ? JSON.parse(new TextDecoder().decode(buildPromptTooLongBody())) as unknown : body; + return move({ + ...back, 'response.copilot.errorBody': body, 'response.provider.output': { + status: tooLong ? 400 : exchange.status, + message: tooLong ? 'prompt is too long' : `Upstream returned ${exchange.status}`, + body: rewritten, + }, + }); + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts new file mode 100644 index 0000000000..2b7edb218d --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts @@ -0,0 +1,71 @@ +import { CLAUDE_AGENT_USER_AGENT } from '../../auth.ts'; +import type { MessagesFacts } from '../../chat-facts.ts'; +import { parseUserIdMetadata } from '../../interceptors/anthropic-messages/detect-claude-code-metadata.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * When Anthropic Messages traffic comes from the Claude Code SDK proxy, VSCode + * Copilot Chat re-tags the native Anthropic Messages upstream call with a different + * intent + user-agent and drops the `copilot-integration-id` it would otherwise + * pin to `vscode-chat`. We mirror that tagging only when the planner selected + * a native Anthropic Messages target and both halves of the Claude Code + * `metadata.user_id` fingerprint are present. + * + * Detection requires BOTH safetyIdentifier AND sessionId so we never apply + * the messages-proxy intent to ordinary chat traffic that happens to share + * one half of the legacy regex. + * + * Sentinel: an empty-string value tells `copilotAuthedFetch` in + * `packages/provider-copilot/src/auth.ts` to delete the named base header. We + * use it to clear `copilot-integration-id` because VSCode Copilot Chat omits + * that header on Claude Code SDK proxy traffic. + * + * Do not put this identity on translated OpenAI Chat Completions / OpenAI Responses targets. + * The real VS Code path forces an Anthropic Messages API request, and caozhiyuan's gateway + * applies the same helper only in its `/v1/messages` Copilot caller. Copilot's + * OpenAI Chat Completions endpoint treats the full messages-proxy + Claude Code UA + no-integration + * shape as `integrator: claude-code`, which can hide non-Claude OpenAI Chat Completions models. + * + * `claude-opus-4-8` is currently excluded: that model's upstream WAF returns + * a generic 403 ("Access to this endpoint is forbidden") whenever a request + * carries the Claude-Code-style user-agent without a `copilot-integration-id` + * header. The same header set is accepted on `claude-opus-4-7`, so the gate + * is a model-id rollout gap on Copilot's side, not a policy choice on ours. + * Skipping the rewrite for 4-8 keeps the default Copilot identity (vscode-chat + * integration id + GitHubCopilotChat UA + conversation-agent intent) in + * place; that path is 200. TODO: remove this skip once Copilot's upstream + * accepts the Claude-Code identity on 4-8. Probed 2026-05-29. + * + * References: + * - https://github.com/microsoft/vscode-copilot-chat/blob/5863f5a7088958050792b5dccbe8b46c6e13eccc/src/extension/chatSessions/claude/node/claudeLanguageModelServer.ts#L479-L516 + * - https://github.com/caozhiyuan/copilot-api/blob/88840ed80000635902b90a35989b1e795d289fdf/src/services/copilot/create-messages.ts#L110-L116 + * - https://github.com/caozhiyuan/copilot-api/blob/88840ed80000635902b90a35989b1e795d289fdf/src/services/copilot/create-chat-completions.ts#L45-L61 + */ +const UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY = new Set(['claude-opus-4-8']); + +export const copilotAnthropicMessagesSetClaudeAgentHeaders = defineStage({ + name: 'copilotAnthropicMessagesSetClaudeAgentHeaders', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers', 'request.provider.model'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + if (UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY.has(payload.model)) { + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + } + + const { safetyIdentifier, sessionId } = parseUserIdMetadata(payload.metadata?.user_id); + if (safetyIdentifier && sessionId) { + headers = replaceHttpHeader(headers, 'x-interaction-type', 'messages-proxy'); + headers = replaceHttpHeader(headers, 'openai-intent', 'messages-proxy'); + headers = replaceHttpHeader(headers, 'user-agent', CLAUDE_AGENT_USER_AGENT); + headers = replaceHttpHeader(headers, 'copilot-integration-id', ''); + } + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-compact-headers.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-compact-headers.ts new file mode 100644 index 0000000000..43592be632 --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-compact-headers.ts @@ -0,0 +1,139 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesMessage, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Claude Code (and OpenCode) periodically asks the model to summarize the + * conversation so the next turn can run against a compacted transcript. VSCode + * Copilot Chat tags those calls with a different intent + interaction-type so + * Copilot's accounting and abuse controls treat the call as a compaction step + * instead of a user-facing turn. Separately, both agents also resume from a + * fresh out-of-context cut by sending an "auto-continue" prompt; VSCode marks + * those as agent-initiated (without the compaction interaction-type) so they + * still bill as continuation rather than a brand-new user turn. We mirror both + * tagging shapes when we detect them, so Copilot-side billing attribution + * stays consistent with what real VSCode users produce. + * + * Detection is purely structural — we never read auth or per-tenant metadata. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/compact.ts + * - https://github.com/caozhiyuan/copilot-api/blob/main/src/routes/messages/preprocess.ts (getCompactType) + * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/api-config.ts (prepareForCompact) + */ + +// Constants ported verbatim from caozhiyuan/copilot-api so detection drift +// stays observable when their values move. +const COMPACT_TEXT_ONLY_GUARD = 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.'; +const COMPACT_SUMMARY_PROMPT_START = 'Your task is to create a detailed summary of the conversation so far'; +const COMPACT_MESSAGE_SECTIONS = ['Pending Tasks:', 'Current Work:'] as const; +const COMPACT_SYSTEM_PROMPT_STARTS = [ + 'You are a helpful AI assistant tasked with summarizing conversations', + 'You are an anchored context summarization assistant for coding sessions.', +] as const; + +// Auto-continue prompts: the first user turn after Claude Code / OpenCode +// resume from an out-of-context cut. The text comes from the agent harness, +// not the human, so VSCode-shaped accounting treats it as agent-initiated. +// See caozhiyuan's `compactAutoContinuePromptStarts` for the source list. +const COMPACT_AUTO_CONTINUE_PROMPT_STARTS = [ + // Claude Code resume prompt. + 'This session is being continued from a previous conversation that ran out of context. The summary below covers the earlier portion of the conversation.', + // OpenCode primary continuation prompt. + 'Continue if you have next steps, or stop and ask for clarification if you are unsure how to proceed.', + // OpenCode media-eviction continuation prompt. + "The previous request exceeded the provider's size limit due to large media attachments. The conversation was compacted and media files were removed from context.", +] as const; + +// Compact-summary and auto-continue prompts are always authored as a fresh +// user turn — the agent harness injects them on the human's behalf. An +// assistant history item whose text happens to start with one of those +// markers (e.g. a client that round-trips the previous request's user turn +// back as assistant history) must not trip compact tagging, so the extractor +// returns empty text for any non-user role. This mirrors caozhiyuan's +// `getCompactCandidateText`: +// https://github.com/caozhiyuan/copilot-api/blob/main/src/routes/messages/preprocess.ts#L94 +// +// `` blocks are Claude Code's own injected reminders that +// the agent never authored; they should never count as compact-summary +// evidence either. +const lastMessageText = (message: AnthropicMessagesMessage): string => { + if (message.role !== 'user') return ''; + if (typeof message.content === 'string') return message.content; + return message.content + .filter((block): block is AnthropicMessagesTextBlock => block.type === 'text') + .map(block => (block.text.startsWith('') ? '' : block.text)) + .filter(text => text.length > 0) + .join('\n\n'); +}; + +const isCompactLastMessage = (message: AnthropicMessagesMessage | undefined): boolean => { + if (!message) return false; + const text = lastMessageText(message); + if (!text) return false; + // All three markers must be present together; the text-only guard alone + // appears in unrelated Claude Code prompts and would over-match. + return text.includes(COMPACT_TEXT_ONLY_GUARD) && text.includes(COMPACT_SUMMARY_PROMPT_START) && COMPACT_MESSAGE_SECTIONS.some(section => text.includes(section)); +}; + +const isAutoContinueLastMessage = (message: AnthropicMessagesMessage | undefined): boolean => { + if (!message) return false; + const text = lastMessageText(message); + if (!text) return false; + return COMPACT_AUTO_CONTINUE_PROMPT_STARTS.some(prefix => text.startsWith(prefix)); +}; + +const startsWithCompactSystemPrompt = (text: string): boolean => COMPACT_SYSTEM_PROMPT_STARTS.some(prefix => text.startsWith(prefix)); + +const isCompactSystemPrompt = (system: string | AnthropicMessagesTextBlock[] | undefined): boolean => { + if (typeof system === 'string') return startsWithCompactSystemPrompt(system); + if (Array.isArray(system)) return system.some(block => startsWithCompactSystemPrompt(block.text)); + return false; +}; + +type CompactClass = 'compact-request' | 'auto-continue' | null; + +// Match caozhiyuan's `getCompactType` priority exactly: last-message +// compact-summary wins, then last-message auto-continue, then system-prompt +// compact-summary. This ordering matters because a single payload could in +// principle match more than one shape, and the COMPACT_REQUEST tagging is +// strictly stronger than the auto-continue tagging. +const classifyCompact = (payload: { messages: AnthropicMessagesMessage[]; system?: string | AnthropicMessagesTextBlock[] }): CompactClass => { + const last = payload.messages.at(-1); + if (isCompactLastMessage(last)) return 'compact-request'; + if (isAutoContinueLastMessage(last)) return 'auto-continue'; + if (isCompactSystemPrompt(payload.system)) return 'compact-request'; + return null; +}; + +export const copilotAnthropicMessagesSetCompactHeaders = defineStage({ + name: 'copilotAnthropicMessagesSetCompactHeaders', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const kind = classifyCompact(payload); + if (kind === 'compact-request') { + headers = replaceHttpHeader(headers, 'x-initiator', 'agent'); + headers = replaceHttpHeader(headers, 'x-interaction-type', 'conversation-compaction'); + // openai-intent stays at `copilotAuthedFetch`'s `conversation-agent` + // default from `packages/provider-copilot/src/auth.ts` — the same value + // caozhiyuan/copilot-api re-pins inside prepareForCompact, so setting it + // here would be a no-op. + } else if (kind === 'auto-continue') { + // Auto-continue gets only the agent-initiator tag; interaction-type stays + // at `copilotAuthedFetch`'s `conversation-agent` default from + // `packages/provider-copilot/src/auth.ts`. + // This mirrors prepareForCompact when compactType === COMPACT_AUTO_CONTINUE: + // it sets x-initiator: agent and leaves x-interaction-type untouched. + headers = replaceHttpHeader(headers, 'x-initiator', 'agent'); + } + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts new file mode 100644 index 0000000000..206aecb4ab --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts @@ -0,0 +1,51 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot's `x-initiator` header distinguishes turns that the human user just + * triggered (`user`) from turns the agent triggered to consume a tool result + * (`agent`). The header gates Copilot-side abuse controls and conversation + * accounting. We classify by the last message: a user turn whose content + * mixes plain blocks with tool_results is still user-initiated; a user turn + * whose content is *only* tool_results is the agent consuming results, and + * any assistant-final turn (which only happens on count-tokens replays) is + * always agent. + * + * The header name is lowercase `x-initiator`; HTTP header names are + * case-insensitive on the wire, so the casing is cosmetic. + * + * Generic in the run-result type so the count_tokens boundary chain + * (`Response`) and the streaming Anthropic Messages boundary chain (`ExecuteResult<...>`) + * can share one definition, matching the established behavior where + * x-initiator is set on every Copilot Anthropic Messages HTTP call. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/master/src/services/copilot/create-chat-completions.ts + */ + +export const copilotAnthropicMessagesSetInitiatorHeader = defineStage({ + name: 'copilotAnthropicMessagesSetInitiatorHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const lastMessage = payload.messages[payload.messages.length - 1]; + let initiator: 'user' | 'agent'; + if (lastMessage?.role !== 'user') { + initiator = 'agent'; + } else if (!Array.isArray(lastMessage.content)) { + initiator = 'user'; + } else { + initiator = lastMessage.content.some(block => block.type !== 'tool_result') ? 'user' : 'agent'; + } + headers = replaceHttpHeader(headers, 'x-initiator', initiator); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts new file mode 100644 index 0000000000..777191a80b --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts @@ -0,0 +1,46 @@ +import { v4 } from 'uuid'; + +import type { MessagesFacts } from '../../chat-facts.ts'; +import { parseUserIdMetadata } from '../../interceptors/anthropic-messages/detect-claude-code-metadata.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot's `x-interaction-id` header threads a conversation through its + * accounting and trace tooling. We hash the raw session identifier through + * SHA-256 and format the first 16 bytes as a UUID v4 string, so the on-wire + * value stays a UUID-shaped opaque identifier rather than leaking the + * upstream client's raw session id. Same input → same UUID, so trace + * correlation across requests still works. + * + * Fires whenever `parseUserIdMetadata` produces a `sessionId`, regardless of + * whether the safety-identifier half is also present. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/api-config.ts (prepareInteractionHeaders, getRootSessionId) + * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/utils.ts#L217 (getRootSessionId) + * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/utils.ts#L230 (getUUID) + */ +const sessionUuid = async (input: string): Promise => { + const data = new TextEncoder().encode(input); + return v4({ random: new Uint8Array(await crypto.subtle.digest('SHA-256', data)) }); +}; + +export const copilotAnthropicMessagesSetInteractionIdHeader = defineStage({ + name: 'copilotAnthropicMessagesSetInteractionIdHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const { sessionId } = parseUserIdMetadata(payload.metadata?.user_id); + if (sessionId) { + headers = replaceHttpHeader(headers, 'x-interaction-id', await sessionUuid(sessionId)); + } + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts new file mode 100644 index 0000000000..c18924ba60 --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts @@ -0,0 +1,50 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesAssistantMessage, AnthropicMessagesUserMessage } from '@floway-dev/protocols/anthropic-messages'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot rejects Anthropic `image` blocks as plain text unless the private + * `copilot-vision-request: true` header is set. Detection must scan the final + * post-mutation payload (after other Anthropic Messages boundary stages have run) + * and cover both the top-level `message.content` and the nested + * `tool_result.content[]` shape; Anthropic allows images in both positions. + * + * Generic in the run-result type because the Copilot provider historically + * applied equivalent vision detection to every Anthropic Messages HTTP exchange (chat + * AND count_tokens). Keeping a single generic stage lets both the streaming + * Anthropic Messages boundary chain (`ExecuteResult<...>`) and the count_tokens chain + * (`Response`) share one definition. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/commit/1f6b98924ae092db9b2010846c32e5cbf10817df + */ +const contentHasImage = (content: AnthropicMessagesUserMessage['content'] | AnthropicMessagesAssistantMessage['content']): boolean => { + if (!Array.isArray(content)) return false; + return content.some(block => { + if (block.type === 'image') return true; + if (block.type === 'tool_result' && Array.isArray(block.content)) { + return block.content.some(inner => inner.type === 'image'); + } + return false; + }); +}; + +export const copilotAnthropicMessagesSetVisionHeader = defineStage({ + name: 'copilotAnthropicMessagesSetVisionHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + if (payload.messages.some(message => contentHasImage(message.content))) { + headers = replaceHttpHeader(headers, 'copilot-vision-request', 'true'); + } + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/strip-cache-control-extensions.ts b/packages/provider-copilot/src/stages/anthropic-messages/strip-cache-control-extensions.ts new file mode 100644 index 0000000000..622cb7c3dd --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/strip-cache-control-extensions.ts @@ -0,0 +1,77 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { mapKeepingIdentity, withKeysChanged } from '@floway-dev/protocols/common'; + +/** + * Two `cache_control` sub-fields are beta extensions to the base + * `CacheControlEphemeral` shape that Copilot's stricter Anthropic-Messages-upstream + * deployments (claude-haiku-4.5, claude-sonnet-4.5/4.6, intermittently + * claude-opus-4.5) reject: + * + * - `scope`: added by Claude Code's `prompt-caching-scope-2025-11-27` beta. + * Copilot returns `cache_control.scope: Extra inputs are not permitted`. + * - `ttl`: added by the `extended-cache-ttl-2025-04-11` beta. Floway does + * not enable that beta for Copilot, so any `ttl` value trips the same + * schema rejection on the body. + * + * Walk every position where `cache_control` may appear — system blocks, + * tools, message content blocks including `tool_use` and `tool_result` — + * and strip both sub-fields, keeping `{ type: 'ephemeral' }` so prompt + * caching still primes on slots that do honour the marker. + * + * The top-level `cache_control` field is handled by + * `withTopLevelCacheControlApplied`, which runs first and either ports it + * onto the last cacheable block (where this stage cleans it) or + * deletes it when no cacheable block exists. + * + * Custom providers that speak Anthropic Messages directly may accept these + * betas natively; this stage is Copilot-only. + * + * References: + * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L2909-L2913 + * - https://github.com/caozhiyuan/copilot-api/issues/143 + * - https://github.com/caozhiyuan/copilot-api/issues/144 + * - https://github.com/caozhiyuan/copilot-api/issues/269 + * - https://github.com/caozhiyuan/copilot-api/commit/ce8224c55933f811abe5bf9ba42f9336a7852997 + */ +/** The block with its cache-control extensions gone, or the same block when it carried none. + * Returning the original is what lets an untouched message ride through by identity. */ +const stripExtensions = (block: T): T => { + const cacheControl = (block as Record).cache_control; + if (!cacheControl || typeof cacheControl !== 'object') return block; + + const { scope: _scope, ttl: _ttl, ...rest } = cacheControl as Record; + return withKeysChanged(block, { cache_control: Object.keys(rest).length > 0 ? rest : undefined }); +}; + +export const copilotAnthropicMessagesStripCacheControlExtensions = defineStage({ + name: 'copilotAnthropicMessagesStripCacheControlExtensions', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + const system = Array.isArray(payload.system) + ? mapKeepingIdentity(payload.system, stripExtensions) + : payload.system; + const tools = payload.tools === undefined ? payload.tools : mapKeepingIdentity(payload.tools, stripExtensions); + const messages = mapKeepingIdentity(payload.messages, message => ( + Array.isArray(message.content) + // The content arrays are a union of per-role block types, so the walk is expressed over + // the shape they share — an object — and handed back at the array's own type. + ? withKeysChanged(message, { content: mapKeepingIdentity(message.content as readonly object[], stripExtensions) }) + : message + )); + + payload = withKeysChanged(payload, { + ...(system === payload.system ? {} : { system }), + ...(tools === payload.tools ? {} : { tools }), + ...(messages === payload.messages ? {} : { messages }), + }); + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts b/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts new file mode 100644 index 0000000000..72c3836850 --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts @@ -0,0 +1,39 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * `eager_input_streaming` is a per-tool property in the Anthropic Messages API + * that enables fine-grained tool input streaming. Copilot's native Anthropic Messages + * target has been observed to reject it with + * `"tools.N.custom.eager_input_streaming: Extra inputs are not permitted"`, so + * strip it only at the Copilot target boundary and leave other providers + * untouched. + * + * References: + * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L1761 + */ + +export const copilotAnthropicMessagesStripEagerInputStreaming = defineStage({ + name: 'copilotAnthropicMessagesStripEagerInputStreaming', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + if (payload.tools) { + payload = { + ...payload, tools: payload.tools.map(tool => { + const { eager_input_streaming: _, ...rest } = tool as typeof tool & { + eager_input_streaming?: unknown; + }; + return rest; + }), + }; + } + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/authenticate-chat.ts b/packages/provider-copilot/src/stages/authenticate-chat.ts new file mode 100644 index 0000000000..d6670768b8 --- /dev/null +++ b/packages/provider-copilot/src/stages/authenticate-chat.ts @@ -0,0 +1,24 @@ +import { getCopilotToken, isCopilotTokenFetchError, type CopilotAuth } from '../auth.ts'; +import type { AuthenticatedChatFacts, CopilotChatFacts } from '../chat-facts.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, secret } from '@floway-dev/pipeline'; +import type { ChatProviderOperation, ProviderChatResponse, ProviderServices } from '@floway-dev/provider'; + +export const authenticateCopilotChat = (auth: CopilotAuth) => defineStage, ProviderChatResponse, ProviderChatResponse, ProviderServices>({ + name: 'authenticateCopilotChat', + through: { + request: { needs: ['request.http.callId', 'request.provider.modelKey'], consumes: [], provides: ['request.copilot.session'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.output'] }, + execute: async (facts, next, use) => { + const call = use.httpCall(facts['request.http.callId']); + let entry; + try { entry = await getCopilotToken(auth.id, auth.githubHost, auth.githubToken, call.fetcher, call.signal); } catch (error) { + if (!isCopilotTokenFetchError(error)) throw error; + const exchange = takeHttpResponse(new Response(error.body, { status: error.status, headers: error.headers })); + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': facts['request.provider.modelKey'], 'response.provider.called': false, 'response.provider.previousCalls': [], 'response.provider.output': null }); + } + return move({ ...await next(move({ ...facts, 'request.copilot.session': { ...entry, token: secret(entry.token) } })) }); + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts new file mode 100644 index 0000000000..c98c5b88ce --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts @@ -0,0 +1,76 @@ + +import { checkWhitespaceOverflow } from '../../interceptors/shared/whitespace-overflow.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +/** + * Copilot has been observed to emit only whitespace (`\r`, `\n`, `\t`) inside + * `function.arguments` deltas until `max_tokens`, never producing valid JSON. + * Detect that pattern per tool call and abort by throwing, so every source + * (native OpenAI Chat Completions, plus Anthropic Messages/Gemini generateContent/OpenAI Responses + * sources translated via OpenAI Chat Completions) + * sees the gateway's standard upstream-error path. + * + * The OpenAI Chat Completions protocol cannot express a stream error in-band: the + * `finish_reason` enum lacks an 'error' value, and the de-facto + * `data: {"error":{...}}` chunk pattern is only recognized by some + * translators (e.g. gemini-generate-content-via-openai-chat-completions calls + * `openaiChatCompletionsErrorPayloadMessage`) but not others + * (e.g. anthropic-messages-via-openai-chat-completions iterates `choices[].delta` and would + * silently drop an error chunk). Throwing keeps the abort semantics uniform + * across every consumer; the source layer converts the thrown error into the + * downstream protocol's native error event. + * + * Lives at the Copilot provider boundary so other OpenAI-compatible providers + * are not slowed by per-delta whitespace inspection. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/commit/4c0d775e1dc6b8648c7ad5f21fb783fc3246facf + * - https://github.com/caozhiyuan/copilot-api/commit/3cdc32c0811469da9eebec5ca3892caf068df542 + */ +const isWhitespaceExceeded = ( + chunk: OpenAIChatCompletionsStreamEvent, + whitespaceByIndex: Map, +): boolean => { + for (const choice of chunk.choices) { + const toolCalls = choice.delta.tool_calls; + if (!toolCalls) continue; + + for (const toolCall of toolCalls) { + const args = toolCall.function?.arguments; + if (!args) continue; + + const current = whitespaceByIndex.get(toolCall.index) ?? 0; + const { count, exceeded } = checkWhitespaceOverflow(args, current); + whitespaceByIndex.set(toolCall.index, count); + if (exceeded) return true; + } + } + return false; +}; + +export const copilotOpenAIChatCompletionsAbortToolWhitespace = defineStage, ProviderChatResponse<'openaiChatCompletions'>, ProviderChatServices>({ + name: 'copilotOpenAIChatCompletionsAbortToolWhitespace', + through: { request: { needs: [], consumes: [], provides: [] }, response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] } }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const output = back['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'stream') return move({ ...back }); + return move({ + ...back, 'response.provider.output': { + ...output, frames: use.recordProtocolFrames((async function* (): AsyncGenerator> { + const whitespaceByIndex = new Map(); + + for await (const frame of output.frames) { + if (frame.type === 'event' && isWhitespaceExceeded(frame.event, whitespaceByIndex)) { + throw new Error('Copilot tool call arguments contained excessive consecutive whitespace, indicating a degenerate response.'); + } + yield frame; + } + })()), + }, + }); + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-chat-completions/attach-cache-control-markers.ts b/packages/provider-copilot/src/stages/openai-chat-completions/attach-cache-control-markers.ts new file mode 100644 index 0000000000..17d9759c4b --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-chat-completions/attach-cache-control-markers.ts @@ -0,0 +1,93 @@ +import type { ChatCompletionsFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; + +/** + * Prime Copilot's content-addressed prompt cache by tagging a small set of + * messages with `copilot_cache_control: { type: 'ephemeral' }`. Selection: + * the first two eligible system messages and the last two eligible + * non-system messages, in original order. + * + * The field is a Copilot-private extension to the OpenAI Chat Completions + * wire shape — kept off the shared `OpenAIChatCompletionsMessage` type so other OpenAI-compatible + * upstreams never see it on their inbound payloads, and attached here through + * a Copilot-local view that widens `OpenAIChatCompletionsMessage` with the marker. + * + * Probe findings (not obvious from the upstream reference): + * + * 1. Cache hits surface ONLY via `usage.prompt_tokens_details.cached_tokens` + * in Copilot's responses. There is no `cache_creation_input_tokens` on + * Copilot regardless of model family. + * 2. The markers are needed only to PRIME the cache. Once a prompt prefix has + * been primed, subsequent calls hitting the same content (byte-for-byte) + * will get a cache hit even when no `copilot_cache_control` is attached. + * The cache is content-addressed, not marker-addressed. + * 3. Copilot internally decides whether to actually cache based on prompt + * size (~1.7k tokens observed as the lower bound). We always attach so + * short prompts pay nothing and long ones get primed automatically. + * 4. The marker is silently accepted by every observed model family + * (gpt-5.4, gpt-5-mini, gpt-4o-mini, claude-sonnet-4.5). No 4xx, no + * schema rejection, no behavioral difference beyond cache priming. + * + * Primes every source path routed through Copilot OpenAI Chat Completions — native + * OpenAI Chat Completions, Anthropic-Messages-via-OpenAI-Chat-Completions, OpenAI-Responses-via-OpenAI-Chat-Completions, and Gemini-generateContent-via-OpenAI-Chat-Completions. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/9be0eb602f1ffee7597741c9af9bc66a68e1a241/src/routes/messages/api-flows.ts#L381-L432 + */ + +export interface CopilotCacheableMessage extends OpenAIChatCompletionsMessage { + copilot_cache_control?: { type: 'ephemeral' }; +} + +const COPILOT_CONTEXT_CACHE_SYSTEM_MARKER_LIMIT = 2; +const COPILOT_CONTEXT_CACHE_NON_SYSTEM_MARKER_LIMIT = 2; +const COPILOT_CONTEXT_CACHE_CONTROL = { type: 'ephemeral' } as const; + +const isEligible = (message: OpenAIChatCompletionsMessage): boolean => { + const { content } = message; + if (typeof content === 'string') return content.length > 0; + return Array.isArray(content) && content.length > 0; +}; + +const selectCacheMarkerIndexes = (messages: readonly OpenAIChatCompletionsMessage[]): number[] => { + const systemIndexes: number[] = []; + for (let i = 0; i < messages.length && systemIndexes.length < COPILOT_CONTEXT_CACHE_SYSTEM_MARKER_LIMIT; i++) { + if (messages[i].role === 'system' && isEligible(messages[i])) systemIndexes.push(i); + } + + const nonSystemIndexes: number[] = []; + for (let i = messages.length - 1; i >= 0 && nonSystemIndexes.length < COPILOT_CONTEXT_CACHE_NON_SYSTEM_MARKER_LIMIT; i--) { + if (messages[i].role !== 'system' && isEligible(messages[i])) nonSystemIndexes.push(i); + } + + return [...new Set([...systemIndexes, ...nonSystemIndexes])].sort((a, b) => a - b); +}; + +export const copilotOpenAIChatCompletionsAttachCacheControlMarkers = defineStage({ + name: 'copilotOpenAIChatCompletionsAttachCacheControlMarkers', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + const marked = new Set(selectCacheMarkerIndexes(payload.messages)); + if (marked.size === 0) return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + payload = { + ...payload, + // A marker is its own object per message, so nothing downstream can carry a change from + // one to another. The messages that were not selected ride through by identity. + messages: payload.messages.map((message, index) => ( + marked.has(index) + ? { ...message, copilot_cache_control: { ...COPILOT_CONTEXT_CACHE_CONTROL } } as CopilotCacheableMessage + : message + )), + }; + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts b/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts new file mode 100644 index 0000000000..cdd83cfe20 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts @@ -0,0 +1,66 @@ +import type { ChatCompletionsFacts } from '../../chat-facts.ts'; +import { memoizedDataUrlCompressor } from '../../interceptors/image-compression.ts'; +import { targetSizeForOpenAIResponsesChat } from '../../interceptors/image-size.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsContentPart, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; +import { type ProviderModelFacts } from '@floway-dev/provider'; +import { isBase64ImageDataUrl } from '@floway-dev/provider'; + +type OpenAIChatCompletionsImagePart = Extract; + +const compressInlineImages = async (payload: OpenAIChatCompletionsPayload, model: ProviderModelFacts): Promise => { + const targets: OpenAIChatCompletionsImagePart[] = []; + for (const message of payload.messages) { + if (!Array.isArray(message.content)) continue; + for (const part of message.content) { + if (part.type === 'image_url' && isBase64ImageDataUrl(part.image_url.url)) targets.push(part); + } + } + + if (targets.length === 0) return payload; + + const compress = memoizedDataUrlCompressor(targetSizeForOpenAIResponsesChat(model.id)); + const compressedUrls = new Map(); + await Promise.all( + targets.map(async target => { + compressedUrls.set(target, await compress(target.image_url.url)); + }), + ); + const hasCompressedImage = (part: OpenAIChatCompletionsContentPart): part is OpenAIChatCompletionsImagePart => + part.type === 'image_url' && compressedUrls.has(part); + const rewriteImage = (part: OpenAIChatCompletionsImagePart): OpenAIChatCompletionsImagePart => { + const url = compressedUrls.get(part); + if (url === undefined) throw new Error('Missing compressed OpenAI Chat Completions image URL'); + return { ...part, image_url: { ...part.image_url, url } }; + }; + return { + ...payload, + messages: payload.messages.map((message): OpenAIChatCompletionsMessage => { + if (!Array.isArray(message.content) || !message.content.some(hasCompressedImage)) return message; + return { + ...message, + content: message.content.map(part => hasCompressedImage(part) ? rewriteImage(part) : part), + }; + }), + }; +}; + +// Recompresses every inline base64 image (`data:image/*;base64,...` in an +// `image_url` part) in the outgoing OpenAI Chat Completions payload to WebP before +// the Copilot upstream call. Remote https image references are left untouched. + +export const copilotOpenAIChatCompletionsCompressImages = defineStage({ + name: 'copilotOpenAIChatCompletionsCompressImages', + through: { + request: { needs: ['request.provider.payload', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + const model = facts['request.provider.model']; + + payload = await compressInlineImages(payload, model); + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-chat-completions/set-initiator-header.ts b/packages/provider-copilot/src/stages/openai-chat-completions/set-initiator-header.ts new file mode 100644 index 0000000000..591a578b37 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-chat-completions/set-initiator-header.ts @@ -0,0 +1,56 @@ +import type { ChatCompletionsFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot's `x-initiator` header distinguishes user-triggered turns from + * agent-triggered tool-result consumption. On OpenAI Chat Completions the + * discriminator is the last message: when its role is `assistant` (model + * replay) or `tool` (a tool result being fed back into the model), the agent + * is driving the turn. + * + * OpenAI Responses tool-output images expose a deliberate translation loss here: + * OpenAI Chat Completions tool messages cannot carry image parts, so translation lifts them into + * a legal user message after the contiguous tool results. The OpenAI Chat Completions wire role + * remains authoritative, and that turn is therefore reported as user-initiated + * even though its image originated in tool output. Preserving the source-side + * provenance would require a contradictory out-of-band signal. + * + * Official clients do not offer one wire-derived rule to copy. VS Code keeps + * image content on a private multimodal `role: "tool"` OpenAI Chat Completions message and lifts + * only for OpenAI Responses, while initiator provenance can travel separately from + * serialized roles. Floway has neither private OpenAI Chat Completions tool-image syntax nor + * client loop metadata; this remains a payload-shape heuristic, and translated + * or synthetic final user messages can therefore be classified differently + * from their source-side turn. + * + * The header name is lowercase `x-initiator`; HTTP header names are + * case-insensitive on the wire, so the casing is cosmetic. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/services/copilot/create-chat-completions.ts#L28-L49 + * - https://github.com/openai/openai-node/blob/61539248cbe04665de68a71e6fd878127ae4db87/src/resources/chat/completions/completions.ts#L1893-L1908 + * - https://github.com/microsoft/vscode-prompt-tsx/blob/86cc3a025fb54b72b0b5be9ddbc786ea16ef4073/src/base/output/openaiConvert.ts#L15-L81 + * - https://github.com/microsoft/vscode/blob/fb5e582d1c8edb9ad0a69e50fe6f508a8c095466/extensions/copilot/src/platform/endpoint/node/responsesApi.ts#L419-L468 + * - https://github.com/microsoft/vscode/blob/fb5e582d1c8edb9ad0a69e50fe6f508a8c095466/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts#L1453-L1474 + */ + +export const copilotOpenAIChatCompletionsSetInitiatorHeader = defineStage({ + name: 'copilotOpenAIChatCompletionsSetInitiatorHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const lastMessage = payload.messages.at(-1); + const agentInitiated = lastMessage?.role === 'assistant' + || lastMessage?.role === 'tool'; + headers = replaceHttpHeader(headers, 'x-initiator', agentInitiated ? 'agent' : 'user'); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-chat-completions/set-vision-header.ts b/packages/provider-copilot/src/stages/openai-chat-completions/set-vision-header.ts new file mode 100644 index 0000000000..8d6a5a428f --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-chat-completions/set-vision-header.ts @@ -0,0 +1,33 @@ +import type { ChatCompletionsFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot's OpenAI Chat Completions endpoint requires the private + * `copilot-vision-request: true` header before it accepts OpenAI-style + * `image_url` content parts; without it images are silently dropped or + * rejected. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/services/copilot/create-chat-completions.ts#L28-L49 + */ + +export const copilotOpenAIChatCompletionsSetVisionHeader = defineStage({ + name: 'copilotOpenAIChatCompletionsSetVisionHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const hasImage = payload.messages.some( + message => Array.isArray(message.content) && message.content.some(part => part.type === 'image_url'), + ); + if (hasImage) headers = replaceHttpHeader(headers, 'copilot-vision-request', 'true'); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts new file mode 100644 index 0000000000..06d9d041c1 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts @@ -0,0 +1,74 @@ + +import { checkWhitespaceOverflow } from '../../interceptors/shared/whitespace-overflow.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +/** + * Copilot has been observed to emit only whitespace (`\r`, `\n`, `\t`) inside + * `response.function_call_arguments.delta` events until `max_tokens`, never + * producing valid JSON arguments. Detect that pattern per function call output + * index and abort the upstream stream before the client times out. + * + * Behaviour: when any single output index's argument deltas accumulate more + * than `MAX_CONSECUTIVE_WHITESPACE` consecutive whitespace characters, emit a + * OpenAI Responses `error` event followed by a done frame, then end the stream. + * The OpenAI Responses source layer surfaces `error` events as a stream failure. + * + * Lives at the Copilot provider boundary so other OpenAI-Responses-capable providers + * are not slowed by per-delta whitespace inspection. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/commit/4c0d775e1dc6b8648c7ad5f21fb783fc3246facf + * - https://github.com/caozhiyuan/copilot-api/commit/3cdc32c0811469da9eebec5ca3892caf068df542 + */ +const ABORT_MESSAGE = 'Tool call arguments contained excessive whitespace, indicating a degenerate response.'; + +const isArgumentsDelta = (event: OpenAIResponsesStreamEvent): event is OpenAIResponsesStreamEvent & { type: 'response.function_call_arguments.delta'; output_index: number; delta: string } => + event.type === 'response.function_call_arguments.delta'; + +const errorEvent = (): OpenAIResponsesStreamEvent => + ({ + type: 'error', + message: ABORT_MESSAGE, + code: 'api_error', + }) as OpenAIResponsesStreamEvent; + +export const copilotOpenAIResponsesAbortToolWhitespace = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'copilotOpenAIResponsesAbortToolWhitespace', + through: { request: { needs: [], consumes: [], provides: [] }, response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] } }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const output = back['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'stream') return move({ ...back }); + return move({ + ...back, 'response.provider.output': { + ...output, frames: use.recordProtocolFrames((async function* (): AsyncGenerator> { + const whitespaceByIndex = new Map(); + + for await (const frame of output.frames) { + if (frame.type !== 'event' || !isArgumentsDelta(frame.event)) { + yield frame; + continue; + } + + const event = frame.event; + const current = whitespaceByIndex.get(event.output_index) ?? 0; + const { count, exceeded } = checkWhitespaceOverflow(event.delta, current); + whitespaceByIndex.set(event.output_index, count); + + if (exceeded) { + use.log.warn('Copilot: infinite whitespace detected in OpenAI Responses function call arguments, aborting stream'); + yield eventFrame(errorEvent()); + yield doneFrame(); + return; + } + + yield frame; + } + })()), + }, + }); + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/compress-images.ts b/packages/provider-copilot/src/stages/openai-responses/compress-images.ts new file mode 100644 index 0000000000..b8e4c0bd4e --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/compress-images.ts @@ -0,0 +1,90 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { memoizedDataUrlCompressor } from '../../interceptors/image-compression.ts'; +import { targetSizeForOpenAIResponsesChat } from '../../interceptors/image-size.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputImage, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; +import { type ProviderModelFacts } from '@floway-dev/provider'; +import { isBase64ImageDataUrl } from '@floway-dev/provider'; + +// Identity provenance prevents a repeated attempt from recompressing our own WebP output. +// A weak side table leaves handed-over image facts immutable and releases their data with them. +const compressedImageUrls = new WeakMap(); +type CompressibleImagePart = OpenAIResponsesInputImage & { image_url: string }; + +const compressInlineImages = async (payload: CanonicalOpenAIResponsesPayload, model: ProviderModelFacts): Promise => { + const targets: Array<{ part: CompressibleImagePart; imageUrl: string }> = []; + for (const item of payload.input) { + const parts = item.type === 'message' + ? item.content + : item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.output : undefined; + if (!Array.isArray(parts)) continue; + for (const part of parts) { + if (part.type !== 'input_image' || typeof part.image_url !== 'string') continue; + const imagePart = part as CompressibleImagePart; + if (compressedImageUrls.get(imagePart) === imagePart.image_url || !isBase64ImageDataUrl(imagePart.image_url)) continue; + targets.push({ part: imagePart, imageUrl: imagePart.image_url }); + } + } + + if (targets.length === 0) return payload; + + const compress = memoizedDataUrlCompressor(targetSizeForOpenAIResponsesChat(model.id)); + const compressedUrls = new Map(); + await Promise.all( + targets.map(async target => { + compressedUrls.set(target.part, await compress(target.imageUrl)); + }), + ); + const hasCompressedImage = (part: OpenAIResponsesInputContent): part is CompressibleImagePart => + part.type === 'input_image' && compressedUrls.has(part as CompressibleImagePart); + const rewriteImage = (part: CompressibleImagePart): CompressibleImagePart => { + const imageUrl = compressedUrls.get(part); + if (imageUrl === undefined) throw new Error('Missing compressed OpenAI Responses image URL'); + const rewritten: CompressibleImagePart = { ...part, image_url: imageUrl }; + compressedImageUrls.set(rewritten, imageUrl); + return rewritten; + }; + const rewriteParts = (parts: TPart[]): TPart[] => + parts.map(part => hasCompressedImage(part) ? rewriteImage(part) as TPart : part); + + return { + ...payload, + input: payload.input.map(item => { + if (item.type === 'message' && Array.isArray(item.content)) { + return item.content.some(hasCompressedImage) + ? { ...item, content: rewriteParts(item.content) } + : item; + } + if ((item.type === 'function_call_output' || item.type === 'custom_tool_call_output') && Array.isArray(item.output)) { + return item.output.some(hasCompressedImage) + ? { ...item, output: rewriteParts(item.output) } + : item; + } + return item; + }), + }; +}; + +// Recompresses every inline base64 image in the outgoing OpenAI Responses payload to +// WebP before the Copilot upstream call. Images appear both as `input_image` +// parts inside message content and inside function/custom tool outputs +// (multimodal tool results, e.g. a screenshot tool). Remote https and file-id +// references are left untouched. Generic in the run-result type so the same +// definition feeds both the streaming `/responses` chain and the non-streaming +// compaction chain. + +export const copilotOpenAIResponsesCompressImages = defineStage({ + name: 'copilotOpenAIResponsesCompressImages', + through: { + request: { needs: ['request.provider.payload', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + const model = facts['request.provider.model']; + + payload = await compressInlineImages(payload, model); + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/fill-empty-namespace-descriptions.ts b/packages/provider-copilot/src/stages/openai-responses/fill-empty-namespace-descriptions.ts new file mode 100644 index 0000000000..a35ecbd8ae --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/fill-empty-namespace-descriptions.ts @@ -0,0 +1,31 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { mapOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses'; + +/** + * Some Copilot data planes reject empty namespace descriptions permitted by + * the current OpenAI schema. Fill the permitted-empty text at the Copilot + * boundary so the same request remains portable across those deployments. + * + * References: + * - https://github.com/openai/openai-openapi/commit/466c74a42f51c02f1927bc666815251dc53845dc + * - https://github.com/caozhiyuan/copilot-api/issues/345 + * - https://github.com/engineersamuel/copilot-proxy-rs/commit/1c23d7237bd1bd7c24045d0d426412851bfd57f6 + */ + +export const copilotOpenAIResponsesFillEmptyNamespaceDescriptions = defineStage({ + name: 'copilotOpenAIResponsesFillEmptyNamespaceDescriptions', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + payload = mapOpenAIResponsesTools(payload, tool => tool.type === 'namespace' && tool.description === '' + ? { ...tool, description: `Tools in the ${tool.name} namespace.` } + : tool); + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/force-store-false.ts b/packages/provider-copilot/src/stages/openai-responses/force-store-false.ts new file mode 100644 index 0000000000..a5dd432286 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/force-store-false.ts @@ -0,0 +1,31 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * Copilot's `/responses` rejects `store: true` with + * `400 {"error":{"message":"store is not supported","code":"unsupported_value","param":"store"}}`. + * Force `store: false` on the outgoing payload once planning has committed to + * the Copilot OpenAI Responses target so the upstream accepts the request. The + * gateway's own stored-items persistence keys off the caller's original `store` + * value captured at parse time and is unaffected by this upstream-only flag. + * + * Generic in the run-result type so the same definition feeds both the + * streaming `/responses` chain and the non-streaming compaction chain — the + * synth-via-trigger compact call also rejects `store: true`. + */ + +export const copilotOpenAIResponsesForceStoreFalse = defineStage({ + name: 'copilotOpenAIResponsesForceStoreFalse', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + payload = { ...payload, store: false }; + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts b/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts new file mode 100644 index 0000000000..2b32136705 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts @@ -0,0 +1,289 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { unwrapCopilotItemId, wrapCopilotItemId } from '../../interceptors/openai-responses/item-id-carrier.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { encodeHex, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +// OpenAI's published examples establish these item-specific prefixes. Keeping +// the Copilot output inventory closed prevents a new upstream item kind from +// leaking its raw id before we have verified its replay behavior. +// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L57042-L59599 +// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L68023-L68281 +// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L68333-L68748 +// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L74970-L75020 +const COPILOT_OUTPUT_ITEM_POLICIES = { + message: { prefix: 'msg', carrier: null }, + reasoning: { prefix: 'rs', carrier: 'encrypted_content' }, + function_call: { prefix: 'fc', carrier: null }, + custom_tool_call: { prefix: 'ctc', carrier: null }, + web_search_call: { prefix: 'ws', carrier: null }, + tool_search_call: { prefix: 'tsc', carrier: null }, + tool_search_output: { prefix: 'tso', carrier: null }, + program: { prefix: 'cm', carrier: 'fingerprint' }, + program_output: { prefix: 'cmo', carrier: null }, + agent_message: { prefix: 'amsg', carrier: 'agent_content' }, + compaction: { prefix: 'cmp', carrier: 'encrypted_content' }, + shell_call: { prefix: 'sh', carrier: null }, + shell_call_output: { prefix: 'sho', carrier: null }, + apply_patch_call: { prefix: 'apc', carrier: null }, +} as const; + +type CopilotOutputItemType = keyof typeof COPILOT_OUTPUT_ITEM_POLICIES; +type CarrierItem = OpenAIResponsesInputItem | OpenAIResponsesOutputItem; + +const copilotOutputItemType = (item: OpenAIResponsesOutputItem): CopilotOutputItemType => { + if (Object.hasOwn(COPILOT_OUTPUT_ITEM_POLICIES, item.type)) return item.type as CopilotOutputItemType; + throw new TypeError(`Unsupported Copilot OpenAI Responses output item type '${item.type}'`); +}; + +const createPublicItemId = (type: CopilotOutputItemType): string => { + const bytes = crypto.getRandomValues(new Uint8Array(16)); + const suffix = encodeHex(bytes); + return `${COPILOT_OUTPUT_ITEM_POLICIES[type].prefix}_${suffix}`; +}; + +const mapCarrierValues = ( + item: TItem, + transform: (value: string) => string, +): TItem => { + if (!Object.hasOwn(COPILOT_OUTPUT_ITEM_POLICIES, item.type)) return item; + const policy = COPILOT_OUTPUT_ITEM_POLICIES[item.type as CopilotOutputItemType]; + switch (policy.carrier) { + case 'encrypted_content': { + const record = item as CarrierItem & { encrypted_content?: unknown }; + return typeof record.encrypted_content === 'string' + ? { ...item, encrypted_content: transform(record.encrypted_content) } as TItem + : item; + } + case 'fingerprint': { + const record = item as CarrierItem & { fingerprint: string }; + return { ...item, fingerprint: transform(record.fingerprint) } as TItem; + } + case 'agent_content': { + const record = item as Extract; + return { + ...item, + content: record.content.map(content => + content.type === 'encrypted_content' && typeof content.encrypted_content === 'string' + ? { ...content, encrypted_content: transform(content.encrypted_content) } + : content), + } as TItem; + } + case null: + return item; + } +}; + +const restoreInputItem = (item: OpenAIResponsesInputItem): OpenAIResponsesInputItem => { + const upstreamItemIds = new Set(); + const restored = mapCarrierValues(item, value => { + const decoded = unwrapCopilotItemId(value); + if (decoded.kind === 'foreign') return value; + upstreamItemIds.add(decoded.id); + return decoded.value; + }); + + if (upstreamItemIds.size === 0) return restored; + if (upstreamItemIds.size > 1) { + throw new TypeError('Copilot OpenAI Responses item carries conflicting upstream ids'); + } + return { ...restored, id: [...upstreamItemIds][0] } as OpenAIResponsesInputItem; +}; + +const restoreInputItemIds = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => ({ + ...payload, + input: payload.input.map(restoreInputItem), +}); + +const carrierValueCount = (item: OpenAIResponsesOutputItem): number => { + let count = 0; + mapCarrierValues(item, value => { + count += 1; + return value; + }); + return count; +}; + +const normalizeObservedItem = (item: OpenAIResponsesOutputItem, publicId: string): OpenAIResponsesOutputItem => { + copilotOutputItemType(item); + if (carrierValueCount(item) === 0) return { ...item, id: publicId } as OpenAIResponsesOutputItem; + + const upstreamId = 'id' in item ? item.id : undefined; + if (typeof upstreamId !== 'string' || upstreamId.length === 0) { + throw new TypeError(`Copilot OpenAI Responses ${item.type} item has replay state but no upstream id`); + } + return { + ...mapCarrierValues(item, value => wrapCopilotItemId(value, upstreamId)), + id: publicId, + } as OpenAIResponsesOutputItem; +}; + +interface TrackedItem { + readonly type: CopilotOutputItemType; + readonly publicId: string; + added: boolean; +} + +interface StreamItemState { + readonly items: Map; +} + +const trackedAt = (state: StreamItemState, outputIndex: number): TrackedItem => { + const tracked = state.items.get(outputIndex); + if (tracked === undefined) throw new TypeError(`Copilot OpenAI Responses event references output_index ${outputIndex} before output_item.added`); + return tracked; +}; + +const trackObservedItem = ( + state: StreamItemState, + outputIndex: number, + item: OpenAIResponsesOutputItem, +): TrackedItem => { + const type = copilotOutputItemType(item); + const existing = state.items.get(outputIndex); + if (existing === undefined) { + const tracked: TrackedItem = { type, publicId: createPublicItemId(type), added: false }; + state.items.set(outputIndex, tracked); + return tracked; + } + if (existing.type !== type) { + throw new TypeError(`Copilot OpenAI Responses output_index ${outputIndex} changed type from ${existing.type} to ${item.type}`); + } + return existing; +}; + +const normalizeResponseOutput = ( + response: OpenAIResponsesResult, + state: StreamItemState, +): OpenAIResponsesResult => { + if (response.output.length === 0) return response; + return { + ...response, + output: response.output.map((item, outputIndex) => { + const tracked = trackObservedItem(state, outputIndex, item); + return normalizeObservedItem(item, tracked.publicId); + }), + }; +}; + +const ITEM_ID_EVENT_TYPES = new Set([ + 'response.content_part.added', + 'response.content_part.done', + 'response.reasoning_summary_part.added', + 'response.reasoning_summary_part.done', + 'response.reasoning_summary_text.delta', + 'response.reasoning_summary_text.done', + 'response.reasoning_text.delta', + 'response.reasoning_text.done', + 'response.output_text.delta', + 'response.output_text.done', + 'response.output_text.annotation.added', + 'response.web_search_call.in_progress', + 'response.web_search_call.searching', + 'response.web_search_call.completed', + 'response.image_generation_call.in_progress', + 'response.image_generation_call.generating', + 'response.image_generation_call.partial_image', + 'response.image_generation_call.completed', + 'response.function_call_arguments.delta', + 'response.function_call_arguments.done', + 'response.custom_tool_call_input.delta', + 'response.custom_tool_call_input.done', + 'response.apply_patch_call_operation_diff.delta', + 'response.apply_patch_call_operation_diff.done', +]); + +const NO_ITEM_ID_EVENT_TYPES = new Set([ + 'response.shell_call_command.added', + 'response.shell_call_command.delta', + 'response.shell_call_command.done', +]); + +const normalizeStreamEvent = (event: OpenAIResponsesStreamEvent, state: StreamItemState): OpenAIResponsesStreamEvent => { + if (event.type === 'response.output_item.added') { + const tracked = trackObservedItem(state, event.output_index, event.item); + if (tracked.added) { + throw new TypeError(`Copilot OpenAI Responses emitted output_item.added twice for output_index ${event.output_index}`); + } + tracked.added = true; + return { ...event, item: normalizeObservedItem(event.item, tracked.publicId) }; + } + + if (event.type === 'response.output_item.done') { + const tracked = trackObservedItem(state, event.output_index, event.item); + return { ...event, item: normalizeObservedItem(event.item, tracked.publicId) }; + } + + if ( + event.type === 'response.queued' + || event.type === 'response.created' + || event.type === 'response.in_progress' + || event.type === 'response.completed' + || event.type === 'response.incomplete' + || event.type === 'response.failed' + ) { + return { ...event, response: normalizeResponseOutput(event.response, state) }; + } + + if (event.type === 'error') return event; + const carrier = event as OpenAIResponsesStreamEvent & { item_id?: unknown; output_index?: unknown }; + const requiresItemId = ITEM_ID_EVENT_TYPES.has(event.type); + const permitsMissingItemId = NO_ITEM_ID_EVENT_TYPES.has(event.type); + if (!requiresItemId && !permitsMissingItemId) { + if (Object.hasOwn(event, 'item') || Object.hasOwn(event, 'response')) { + throw new TypeError(`Unsupported Copilot OpenAI Responses stream event type '${event.type}'`); + } + if (!Object.hasOwn(carrier, 'item_id')) return event; + } else if (permitsMissingItemId && !Object.hasOwn(carrier, 'item_id')) { + return event; + } + if (typeof carrier.item_id !== 'string') { + const reason = requiresItemId ? 'is missing item_id' : 'carries an invalid item_id extension'; + throw new TypeError(`Copilot OpenAI Responses event '${event.type}' ${reason}`); + } + if (typeof carrier.output_index !== 'number') { + if (!requiresItemId) { + throw new TypeError(`Copilot OpenAI Responses event '${event.type}' carries an invalid item_id extension`); + } + throw new TypeError(`Copilot OpenAI Responses event '${event.type}' carries item_id without output_index`); + } + return { ...carrier, item_id: trackedAt(state, carrier.output_index).publicId } as OpenAIResponsesStreamEvent; +}; + +const normalizeFrames = async function* ( + frames: AsyncIterable>, +): AsyncGenerator> { + const state: StreamItemState = { items: new Map() }; + for await (const frame of frames) { + yield frame.type === 'event' + ? { ...frame, event: normalizeStreamEvent(frame.event, state) } + : frame; + } +}; + +const normalizeCompactionResult = (response: OpenAIResponsesCompactionResult): OpenAIResponsesCompactionResult => ({ + ...response, + output: response.output.map(item => { + if (item.type !== 'compaction') return item; + return normalizeObservedItem(item, createPublicItemId('compaction')); + }), +}); + +export const copilotOpenAIResponsesItemIdMembrane = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'copilotOpenAIResponsesItemIdMembrane', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts, 'request.provider.payload': restoreInputItemIds(facts['request.provider.payload']) })); + const output = back['response.provider.output']; + if (output === null || !('kind' in output)) return move({ ...back }); + return move({ + ...back, 'response.provider.output': output.kind === 'stream' + ? { ...output, frames: use.recordProtocolFrames(normalizeFrames(output.frames)) } + : { ...output, body: normalizeCompactionResult(output.body) }, + }); + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/set-initiator-header.ts b/packages/provider-copilot/src/stages/openai-responses/set-initiator-header.ts new file mode 100644 index 0000000000..31f3608767 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/set-initiator-header.ts @@ -0,0 +1,49 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot's `x-initiator` header distinguishes user-triggered turns from + * agent-triggered tool-result consumption. On OpenAI Responses the discriminator is + * the last input item — but the set of "agent" shapes is broader than just + * `function_call_output`: + * + * - Tool-output-style items lack a `role` field entirely + * (`function_call_output`, `custom_tool_call_output`, `tool_search_output`, + * plus any future hosted-tool output shape). Classify them as agent. + * - An assistant message replayed back into `input` is also agent-driven. + * + * Role-bearing user / system / developer items — including the canonical + * `additional_tools` developer item — and an empty input mean initiator = user. + * + * The header name is lowercase `x-initiator`; HTTP header names are + * case-insensitive on the wire, so the casing is cosmetic. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/cd8207cb70ede07771bf37a04accfbf2af76d980/src/routes/responses/utils.ts#L75-L87 + * (`hasAgentInitiator`) + */ + +export const copilotOpenAIResponsesSetInitiatorHeader = defineStage({ + name: 'copilotOpenAIResponsesSetInitiatorHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const lastItem: OpenAIResponsesInputItem | undefined = payload.input.at(-1); + const role = lastItem === undefined ? undefined : (lastItem as { role?: unknown }).role; + const initiator: 'user' | 'agent' = lastItem !== undefined + && (role === undefined || role === null || role === '' || role === 'assistant') + ? 'agent' + : 'user'; + headers = replaceHttpHeader(headers, 'x-initiator', initiator); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/set-vision-header.ts b/packages/provider-copilot/src/stages/openai-responses/set-vision-header.ts new file mode 100644 index 0000000000..67457ead26 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/set-vision-header.ts @@ -0,0 +1,39 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot's OpenAI Responses endpoint requires the private + * `copilot-vision-request: true` header to accept image inputs. Canonical + * OpenAI Responses carries `input_image` blocks in message content and in multimodal + * function/custom tool output arrays, so all three containers participate in + * the same detection rule. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/lib/api-config.ts#L248-L258 + * - https://github.com/caozhiyuan/copilot-api/blob/cd8207cb70ede07771bf37a04accfbf2af76d980/src/routes/responses/utils.ts#L176-L201 + */ +const itemHasImage = (item: OpenAIResponsesInputItem): boolean => { + const content = item.type === 'message' + ? item.content + : item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.output : undefined; + return Array.isArray(content) && content.some(part => part.type === 'input_image'); +}; + +export const copilotOpenAIResponsesSetVisionHeader = defineStage({ + name: 'copilotOpenAIResponsesSetVisionHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + if (payload.input.some(itemHasImage)) headers = replaceHttpHeader(headers, 'copilot-vision-request', 'true'); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/strip-image-generation.ts b/packages/provider-copilot/src/stages/openai-responses/strip-image-generation.ts new file mode 100644 index 0000000000..8d169450bd --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/strip-image-generation.ts @@ -0,0 +1,76 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { collectOpenAIResponsesTools, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesTool, type OpenAIResponsesToolChoice } from '@floway-dev/protocols/openai-responses'; + +/** + * A Copilot gateway filters public `image_generation` from Responses requests, + * while OpenAI supports it. Apply this provider-specific rule after target + * selection to every declaration carrier and selector, retaining unrelated + * tools in their declaration containers and relative order. + * + * https://developers.openai.com/api/docs/guides/tools-image-generation + * https://github.com/caozhiyuan/copilot-api/blob/5d37d5b1ac6566c935a5c26d046396ee5fa423cc/src/routes/responses/handler.ts#L187-L204 + */ +const isImageGenerationTool = (tool: OpenAIResponsesTool): boolean => tool.type === 'image_generation'; + +const isImageGenerationToolChoice = (choice: OpenAIResponsesToolChoice | null | undefined): boolean => + typeof choice === 'object' && choice !== null && choice.type === 'image_generation'; + +const withoutToolChoice = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { + const { tool_choice: _choice, ...rest } = payload; + return rest; +}; + +export const stripImageGenerationFromPayload = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { + let removedTool = false; + const filter = (tools: OpenAIResponsesTool[]): OpenAIResponsesTool[] => { + const filtered = tools.filter(tool => { + const drop = isImageGenerationTool(tool); + removedTool ||= drop; + return !drop; + }); + return filtered.length === tools.length ? tools : filtered; + }; + let rewritten = payload; + if (Array.isArray(payload.tools)) { + const tools = filter(payload.tools); + if (tools.length === 0) { + const { tools: _tools, ...rest } = rewritten; + rewritten = rest; + } else if (tools !== payload.tools) rewritten = { ...rewritten, tools }; + } + const input = payload.input.map(item => { + if (item.type !== 'additional_tools' && item.type !== 'tool_search_output') return item; + const tools = filter(item.tools); + return tools === item.tools ? item : { ...item, tools }; + }); + if (input.some((item, index) => item !== payload.input[index])) rewritten = { ...rewritten, input }; + const choice = rewritten.tool_choice; + if (isImageGenerationToolChoice(choice)) { + return collectOpenAIResponsesTools(rewritten).length === 0 + ? withoutToolChoice(rewritten) : { ...rewritten, tool_choice: 'none' }; + } + if (typeof choice === 'object' && choice !== null && choice.type === 'allowed_tools') { + const allowed = choice.tools.filter(tool => tool.type !== 'image_generation'); + if (allowed.length === choice.tools.length) return rewritten; + return { ...rewritten, tool_choice: allowed.length === 0 && choice.mode === 'auto' ? 'none' : { ...choice, tools: allowed } }; + } + // Filtering a required choice must not expose tools outside its declared subset. + if (removedTool && choice === 'required' && collectOpenAIResponsesTools(rewritten).length === 0) return withoutToolChoice(rewritten); + return rewritten; +}; + +export const copilotOpenAIResponsesStripImageGeneration = defineStage({ + name: 'copilotOpenAIResponsesStripImageGeneration', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + payload = stripImageGenerationFromPayload(payload); + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/strip-service-tier.ts b/packages/provider-copilot/src/stages/openai-responses/strip-service-tier.ts new file mode 100644 index 0000000000..645198b77b --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/strip-service-tier.ts @@ -0,0 +1,36 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * Copilot has no request-side `service_tier`: `/responses` answers any value + * of the field with HTTP 400 + * `{"code":"unsupported_value","param":"service_tier"}`, on a base id and a + * `-fast` id alike. The tier reaches the upstream as the raw model id + * model selection selected instead, so by the time this runs the value + * has already been consumed and only the unusable field is left to drop. + * Strip it only after planning has committed to the OpenAI Responses target + * so source-side behavior and telemetry still see the caller's original + * request. Generic in the run-result type so the same definition feeds both + * the streaming `/responses` chain and the non-streaming compaction chain. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/commit/f7835a44f06976cab874700e4d94a5f5c0379369 + * - https://platform.openai.com/docs/api-reference/responses/create + */ + +export const copilotOpenAIResponsesStripServiceTier = defineStage({ + name: 'copilotOpenAIResponsesStripServiceTier', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + const { service_tier: _, ...withoutTier } = payload; + payload = withoutTier; + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/prepare-chat-request.ts b/packages/provider-copilot/src/stages/prepare-chat-request.ts new file mode 100644 index 0000000000..77d9c5250e --- /dev/null +++ b/packages/provider-copilot/src/stages/prepare-chat-request.ts @@ -0,0 +1,28 @@ +import { copilotRequestHeaders } from '../auth.ts'; +import type { AuthenticatedChatFacts, CopilotChatHttpFacts } from '../chat-facts.ts'; +import { COMPACTION_TRIGGER } from '../compaction.ts'; +import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ChatProviderOperation, ProviderResponse, ProviderServices } from '@floway-dev/provider'; + +export const prepareCopilotChatRequest = (operation: ChatProviderOperation) => defineStage({ + name: `prepareCopilot${operation}`, + through: { + request: { needs: [...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : []), 'request.provider.payload', 'request.provider.modelKey', 'request.http.headers', 'request.http.callId', 'request.copilot.session'], consumes: ['request.provider.model', 'request.provider.payload', 'request.copilot.session', 'request.http.headers'], provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const { 'request.provider.payload': payload, 'request.copilot.session': session, 'request.provider.model': _, ...rest } = facts; + const admitted = facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value]); + if (operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens') { + const beta = facts['request.provider.anthropicBeta']!; + for (let i = admitted.length - 1; i >= 0; i--) if (admitted[i]![0].toLowerCase() === 'anthropic-beta') admitted.splice(i, 1); + if (beta.length > 0) admitted.push(['anthropic-beta', beta.join(',')]); + } + const headers = [...copilotRequestHeaders({ ...session, token: session.token.reveal() }, undefined, admitted)].map(([name, value]): [string, string | Secret] => [name, name.toLowerCase() === 'authorization' ? secret(value) : value]); + const path = operation === 'openaiChatCompletions' ? '/chat/completions' : operation === 'anthropicMessages' ? '/v1/messages' : operation === 'anthropicMessagesCountTokens' ? '/v1/messages/count_tokens' : '/responses'; + const compact = 'request.provider.responsesAction' in facts && facts['request.provider.responsesAction'] === 'compact'; + const body = { ...payload, model: facts['request.provider.modelKey'], ...(operation === 'anthropicMessagesCountTokens' ? {} : { stream: !compact }), ...(compact ? { input: [...(payload as CanonicalOpenAIResponsesPayload).input, COMPACTION_TRIGGER] } : {}) }; + return move({ ...await next(move({ ...rest, 'request.http.url': `${session.baseUrl}${path}`, 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': body, 'request.http.encoding': 'json' })) }); + }, +}); diff --git a/packages/provider-copilot/src/stages/prepare-request.ts b/packages/provider-copilot/src/stages/prepare-request.ts index 5d33ca5bfc..c8f38aefab 100644 --- a/packages/provider-copilot/src/stages/prepare-request.ts +++ b/packages/provider-copilot/src/stages/prepare-request.ts @@ -1,20 +1,19 @@ import { copilotRequestHeaders } from '../auth.ts'; import { copilotOpenAIEmbeddingsBody, rawModelFor } from '../operation-model.ts'; import type { CopilotRequest, CopilotHttpFacts } from '../pipeline-facts.ts'; -import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; export const prepareCopilotEmbeddings = () => { - const prepare = defineStage({ + const prepare = defineStage({ name: 'prepareCopilotOpenAIEmbeddings', through: { request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.copilot.session'], consumes: ['request.provider.model', 'request.provider.payload', 'request.copilot.session', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.copilot.modelKey'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.copilot.modelKey', 'request.provider.modelKey'], }, - response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next) => { const { 'request.provider.model': model, 'request.provider.payload': payload, 'request.copilot.session': session, ...rest } = facts; @@ -25,13 +24,14 @@ export const prepareCopilotEmbeddings = () => { const back = await next(move({ ...rest, 'request.copilot.modelKey': modelKey, + 'request.provider.modelKey': modelKey, 'request.http.url': `${session.baseUrl}/embeddings`, 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': { ...copilotOpenAIEmbeddingsBody(payload), model: modelKey }, 'request.http.encoding': 'json' as const, })); - return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + return move({ ...back }); }, }); return prepare; diff --git a/packages/provider-copilot/src/stages/reshape-compaction.ts b/packages/provider-copilot/src/stages/reshape-compaction.ts new file mode 100644 index 0000000000..e9e0b1904f --- /dev/null +++ b/packages/provider-copilot/src/stages/reshape-compaction.ts @@ -0,0 +1,19 @@ +import type { ResponsesFacts } from '../chat-facts.ts'; +import { compactionResponse } from '../compaction.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const reshapeCopilotCompaction = defineStage, ProviderChatResponse<'openaiResponsesCompact'>>({ + name: 'reshapeCopilotCompaction', + through: { + request: { needs: ['request.provider.payload'], consumes: [], provides: [] }, + response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] }, + }, + execute: async (facts, next) => { + const back = await next(move({ ...facts })); + const output = back['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'value') return move({ ...back }); + return move({ ...back, 'response.provider.output': { ...output, body: compactionResponse(facts['request.provider.payload'].input, output.body as unknown as OpenAIResponsesResult) } }); + }, +}); diff --git a/packages/provider-copilot/src/stages/select-chat-model.ts b/packages/provider-copilot/src/stages/select-chat-model.ts new file mode 100644 index 0000000000..bc7a5b424b --- /dev/null +++ b/packages/provider-copilot/src/stages/select-chat-model.ts @@ -0,0 +1,46 @@ +import type { CopilotChatFacts } from '../chat-facts.ts'; +import { CONTEXT_1M_BETA, copilotModelSupportsFastVariant } from '../model-selection.ts'; +import { rawModelFor } from '../operation-model.ts'; +import type { CopilotRawModel } from '../types.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, type Handed } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { isFastServiceTier } from '@floway-dev/protocols/common'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ChatProviderOperation, ProviderChatResponse, ProviderOperationRequest, ProviderRequest, ProviderOperationPayloads } from '@floway-dev/provider'; + +export const selectCopilotChatModel = (operation: O) => defineStage & { 'request.provider.anthropicBeta'?: readonly string[] }, CopilotChatFacts, ProviderChatResponse, ProviderChatResponse, ProviderChatResponse>({ + name: `selectCopilot${operation}`, + through: { + request: { needs: [...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], consumes: ['request.provider.payload'], provides: ['request.provider.payload', 'request.provider.modelKey', ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : [])] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.output'] }, + execute: async (facts, next) => { + const model = facts['request.provider.model']; + const payload = facts['request.provider.payload']; + let modelKey: string; + if (operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens') { + const body = payload as Omit; + if (operation === 'anthropicMessages' && body.speed === 'fast' && !copilotModelSupportsFastVariant((model.providerData as { rawModels: CopilotRawModel[] }).rawModels)) { + // Anthropic Fast Mode requires a supporting model rather than silent downgrade. + // https://docs.claude.com/en/build-with-claude/fast-mode + const exchange = takeHttpResponse(Response.json({ type: 'error', error: { type: 'invalid_request_error', message: `'${model.id}' does not support the \`speed\` parameter.` } }, { status: 400 })); + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': model.id, 'response.provider.called': false, 'response.provider.previousCalls': [], 'response.provider.output': null }); + } + modelKey = rawModelFor(model, 'anthropicMessages', { + context1m: (facts as ProviderOperationRequest<'anthropicMessages'>)['request.provider.anthropicBeta'].includes(CONTEXT_1M_BETA), + reasoningEffort: body.output_config?.effort, + fast: operation === 'anthropicMessages' && body.speed === 'fast', + }).id; + } else if (operation === 'openaiChatCompletions') { + const body = payload as Omit; + modelKey = rawModelFor(model, 'openaiChatCompletions', { reasoningEffort: body.reasoning_effort && body.reasoning_effort !== 'none' ? body.reasoning_effort : undefined }).id; + } else { + const body = payload as Omit; + modelKey = rawModelFor(model, 'openaiResponses', { reasoningEffort: body.reasoning?.effort && body.reasoning.effort !== 'none' ? body.reasoning.effort : undefined, fast: isFastServiceTier(body.service_tier) }).id; + } + return move({ ...await next(move({ ...facts, 'request.provider.payload': { ...payload, model: model.id }, 'request.provider.modelKey': modelKey, ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? { 'request.provider.responsesAction': operation === 'openaiResponsesCompact' ? 'compact' as const : 'generate' as const } : {}) }) as Handed) }); + }, +}); diff --git a/packages/provider-custom/__tests__/stages/pipelines_test.ts b/packages/provider-custom/__tests__/stages/pipelines_test.ts new file mode 100644 index 0000000000..8b9949acb6 --- /dev/null +++ b/packages/provider-custom/__tests__/stages/pipelines_test.ts @@ -0,0 +1,58 @@ +import { expect, test } from 'vitest'; + +import { createCustomPipelines } from '../../src/pipelines.ts'; +import { move, run, type Event } from '@floway-dev/pipeline'; +import { providerModelFacts, type ProviderChatServices, type ProviderOperationRequest } from '@floway-dev/provider'; +import { jsonResponse, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; + +const pipelines = createCustomPipelines({ baseUrl: 'https://custom.example', authStyle: 'bearer', apiKey: 'secret-key', endpoints: {}, ingressHeadersRules: [{ key: 'x-many', value: 'first' }, { key: 'x-many', value: 'second' }], pathOverrides: { '/responses': '/overridden' }, modelsFetch: { enabled: false }, models: [] }); +const services: ProviderChatServices = { httpCall: () => ({ ...noopUpstreamCallOptions(), signal: undefined }), recordProtocolFrames: frames => frames }; + +test('custom compact chain derives URL and wire payload from immutable action facts', async () => { + const pipeline = pipelines.openaiResponsesCompact; + if (pipeline === undefined) throw new Error('Missing compact chain'); + const input: ProviderOperationRequest<'openaiResponsesCompact'> = move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: 'wire-model' })), + 'request.provider.payload': { input: [{ type: 'message', role: 'user', content: 'hello' }], store: true, max_output_tokens: 30 }, + 'request.http.callId': 1, + 'request.http.headers': [], + }); + await withMockedFetch(async request => { + expect(request.url).toBe('https://custom.example/overridden/compact'); + expect(request.headers.get('authorization')).toBe('Bearer secret-key'); + expect(request.headers.get('x-many')).toBe('first, second'); + expect(await request.json()).toEqual({ model: 'wire-model', input: [{ type: 'message', role: 'user', content: 'hello' }] }); + return jsonResponse({ object: 'response.compaction', output: [] }); + }, async () => { + const events: Event[] = []; + const executed = await run(pipeline, input, { ...services, dump: (event: Event) => { events.push(event); } }); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.entered', facts: expect.objectContaining({ 'request.provider.responsesAction': 'compact' }) })); + expect(executed.facts['response.provider.called']).toBe(true); + expect(executed.facts['response.provider.output']).toEqual({ kind: 'value', body: { object: 'response.compaction', output: [] } }); + expect(input['request.provider.payload'].store).toBe(true); + await executed.drain(); + }); +}); + +test('count_tokens forwards typed beta intent and retains raw HTTP response facts', async () => { + const pipeline = pipelines.anthropicMessagesCountTokens; + if (pipeline === undefined) throw new Error('Missing count chain'); + const input: ProviderOperationRequest<'anthropicMessagesCountTokens'> = move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: 'wire-model' })), + 'request.provider.payload': { max_tokens: 12, messages: [] }, + 'request.provider.anthropicBeta': ['future-beta'], + 'request.http.callId': 1, + 'request.http.headers': [['anthropic-beta', 'discard-ordinary-header']], + }); + await withMockedFetch(async request => { + expect(request.url).toBe('https://custom.example/v1/messages/count_tokens'); + expect(request.headers.get('anthropic-beta')).toBe('future-beta'); + expect(await request.json()).toEqual({ model: 'wire-model', max_tokens: 12, messages: [] }); + return jsonResponse({ input_tokens: 19 }); + }, async () => { + const executed = await run(pipeline, input, services); + expect(executed.facts['response.provider.output']).toEqual({ kind: 'value', body: { input_tokens: 19 } }); + expect(executed.facts['response.http.exchange']).toMatchObject({ type: 'response', status: 200 }); + await executed.drain(); + }); +}); diff --git a/packages/provider-custom/src/pipelines.ts b/packages/provider-custom/src/pipelines.ts index ce3438c8d0..b67c1d6c80 100644 --- a/packages/provider-custom/src/pipelines.ts +++ b/packages/provider-custom/src/pipelines.ts @@ -2,14 +2,20 @@ import type { CustomUpstreamConfig } from './config.ts'; import { prepareCustomRequest } from './stages/prepare-request.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createCustomPipelines = (config: CustomUpstreamConfig): ProviderPipelines => ({ - alphaSearch: compose, ProviderOperationResponse<'alphaSearch'>>('custom.alphaSearch', [prepareCustomRequest(config, 'alphaSearch'), http]), - openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('custom.openaiCompletions', [prepareCustomRequest(config, 'openaiCompletions'), http]), - openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('custom.openaiEmbeddings', [prepareCustomRequest(config, 'openaiEmbeddings'), http]), - openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('custom.openaiImagesGenerations', [prepareCustomRequest(config, 'openaiImagesGenerations'), http]), - openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('custom.openaiImagesEdits', [prepareCustomRequest(config, 'openaiImagesEdits'), http]), - openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('custom.openaiAudioTranscriptions', [prepareCustomRequest(config, 'openaiAudioTranscriptions'), http]), - rerank: compose, ProviderOperationResponse<'rerank'>>('custom.rerank', [prepareCustomRequest(config, 'rerank'), http]), + alphaSearch: compose, ProviderOperationResponse<'alphaSearch'>>('custom.alphaSearch', [prepareCustomRequest(config, 'alphaSearch'), observeProviderCall, http]), + openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('custom.openaiChatCompletions', [decodeProviderResponse('openaiChatCompletions'), prepareCustomRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('custom.openaiResponses', [selectProviderResponsesAction('generate'), decodeProviderResponse('openaiResponses'), prepareCustomRequest(config, 'openaiResponses'), observeProviderCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('custom.openaiResponsesCompact', [selectProviderResponsesAction('compact'), decodeProviderResponse('openaiResponsesCompact'), prepareCustomRequest(config, 'openaiResponsesCompact'), observeProviderCall, http]), + anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('custom.anthropicMessages', [decodeProviderResponse('anthropicMessages'), prepareCustomRequest(config, 'anthropicMessages'), observeProviderCall, http]), + anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('custom.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareCustomRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('custom.openaiCompletions', [prepareCustomRequest(config, 'openaiCompletions'), observeProviderCall, http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('custom.openaiEmbeddings', [prepareCustomRequest(config, 'openaiEmbeddings'), observeProviderCall, http]), + openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('custom.openaiImagesGenerations', [prepareCustomRequest(config, 'openaiImagesGenerations'), observeProviderCall, http]), + openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('custom.openaiImagesEdits', [prepareCustomRequest(config, 'openaiImagesEdits'), observeProviderCall, http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('custom.openaiAudioTranscriptions', [prepareCustomRequest(config, 'openaiAudioTranscriptions'), observeProviderCall, http]), + rerank: compose, ProviderOperationResponse<'rerank'>>('custom.rerank', [prepareCustomRequest(config, 'rerank'), observeProviderCall, http]), }); diff --git a/packages/provider-custom/src/stages/prepare-request.ts b/packages/provider-custom/src/stages/prepare-request.ts index b8c7f5dfc9..cb108b6ce5 100644 --- a/packages/provider-custom/src/stages/prepare-request.ts +++ b/packages/provider-custom/src/stages/prepare-request.ts @@ -1,12 +1,14 @@ import type { CustomPathOverrideKey, CustomUpstreamConfig } from '../config.ts'; -import { type HttpHeaders, type HttpRequestFacts, type HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { type HttpHeaders, type HttpRequestFacts } from '@floway-dev/http/pipeline'; import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; import type { RerankTarget } from '@floway-dev/protocols/common'; +import { toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { DEFAULT_RERANK_PATHS, serializeRerankRequest } from '@floway-dev/protocols/rerank'; import { joinBaseAndPath, mergeHttpHeaders, prepareOpenAIImagesEditsBody, replaceHttpHeader, withHttpContentType, type ProviderOperation, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; interface CustomHttpFacts extends HttpRequestFacts { + 'request.provider.modelKey': string; 'request.custom.modelKey': string; 'request.custom.path': string; } @@ -14,6 +16,11 @@ interface CustomHttpFacts extends HttpRequestFacts { type CustomResponse = ProviderResponse & { 'response.provider.rerankTarget'?: RerankTarget }; const PATHS: Partial> = { + openaiChatCompletions: '/chat/completions', + openaiResponses: '/responses', + openaiResponsesCompact: '/responses', + anthropicMessages: '/messages', + anthropicMessagesCountTokens: '/messages', alphaSearch: '/alpha/search', openaiCompletions: '/completions', openaiEmbeddings: '/embeddings', @@ -40,18 +47,18 @@ const resolvedHeaders = (config: CustomUpstreamConfig, incoming: HttpHeaders): H }; export const prepareCustomRequest = (config: CustomUpstreamConfig, operation: O) => { - const prepare = defineStage, CustomHttpFacts, HttpResponseFacts, CustomResponse, ProviderServices>({ + const prepare = defineStage & { 'request.provider.anthropicBeta'?: readonly string[]; 'request.provider.responsesAction': 'generate' | 'compact' }, CustomHttpFacts, CustomResponse, CustomResponse, ProviderServices>({ name: `prepareCustom${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, through: { request: { - needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + needs: [...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : [])], consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.custom.modelKey', 'request.custom.path'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.provider.modelKey', 'request.custom.modelKey', 'request.custom.path'], }, response: { - needs: ['response.http.exchange'], + needs: [], consumes: [], - provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', ...(operation === 'rerank' ? ['response.provider.rerankTarget' as const] : [])], + provides: [...(operation === 'rerank' ? ['response.provider.rerankTarget' as const] : [])], }, }, execute: async (facts, next) => { @@ -85,7 +92,11 @@ export const prepareCustomRequest = (config: Custom default: { const key = PATHS[operation]!; path = config.pathOverrides?.[key] ?? `/v1${key}`; - body = { ...payload, model: modelKey }; + if ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact') path += '/compact'; + if (operation === 'anthropicMessagesCountTokens') path += '/count_tokens'; + body = (operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact' + ? { ...toCompactPayloadShape(payload as ProviderOperationPayloads['openaiResponsesCompact']), model: modelKey } + : { ...payload, ...(operation === 'openaiChatCompletions' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' || operation === 'anthropicMessages' ? { stream: true } : {}), model: modelKey }; } } let base: HttpHeaders = []; @@ -96,9 +107,15 @@ export const prepareCustomRequest = (config: Custom base = replaceHttpHeader(base, 'Authorization', secret(`Bearer ${config.apiKey}`)); } base = withHttpContentType(base, body, encoding); - const headers = mergeHttpHeaders(base, resolvedHeaders(config, facts['request.http.headers'])); + let headers = mergeHttpHeaders(base, resolvedHeaders(config, facts['request.http.headers'])); + if ('request.provider.anthropicBeta' in facts) { + const beta = facts['request.provider.anthropicBeta'] as readonly string[]; + headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); + if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; + } const back = await next(move({ ...rest, + 'request.provider.modelKey': modelKey, 'request.custom.modelKey': modelKey, 'request.custom.path': path, 'request.http.url': joinBaseAndPath(config.baseUrl, path), @@ -107,7 +124,7 @@ export const prepareCustomRequest = (config: Custom 'request.http.body': body, 'request.http.encoding': encoding, })); - return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [], ...(target === undefined ? {} : { 'response.provider.rerankTarget': target }) }); + return move({ ...back, ...(target === undefined ? {} : { 'response.provider.rerankTarget': target }) }); }, }); return prepare; diff --git a/packages/provider-ollama/src/pipelines.ts b/packages/provider-ollama/src/pipelines.ts index 495674a2ab..b64a040aa0 100644 --- a/packages/provider-ollama/src/pipelines.ts +++ b/packages/provider-ollama/src/pipelines.ts @@ -4,13 +4,19 @@ import { prepareOllamaRequest } from './stages/prepare-request.ts'; import type { OllamaUpstreamState } from './state.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createOllamaPipelines = (upstreamId: string, config: OllamaUpstreamConfig, state: OllamaUpstreamState): ProviderPipelines => { const observe = observeOllamaAccount(upstreamId, config, state); return { - openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('ollama.openaiCompletions', [observe, prepareOllamaRequest(config, 'openaiCompletions'), http]), - openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('ollama.openaiEmbeddings', [observe, prepareOllamaRequest(config, 'openaiEmbeddings'), http]), - openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('ollama.openaiAudioTranscriptions', [observe, prepareOllamaRequest(config, 'openaiAudioTranscriptions'), http]), + openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('ollama.openaiChatCompletions', [observe, decodeProviderResponse('openaiChatCompletions'), prepareOllamaRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('ollama.openaiResponses', [observe, selectProviderResponsesAction('generate'), decodeProviderResponse('openaiResponses'), prepareOllamaRequest(config, 'openaiResponses'), observeProviderCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('ollama.openaiResponsesCompact', [observe, selectProviderResponsesAction('compact'), decodeProviderResponse('openaiResponsesCompact'), prepareOllamaRequest(config, 'openaiResponsesCompact'), observeProviderCall, http]), + anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('ollama.anthropicMessages', [observe, decodeProviderResponse('anthropicMessages'), prepareOllamaRequest(config, 'anthropicMessages'), observeProviderCall, http]), + anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('ollama.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareOllamaRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('ollama.openaiCompletions', [observe, prepareOllamaRequest(config, 'openaiCompletions'), observeProviderCall, http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('ollama.openaiEmbeddings', [observe, prepareOllamaRequest(config, 'openaiEmbeddings'), observeProviderCall, http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('ollama.openaiAudioTranscriptions', [observe, prepareOllamaRequest(config, 'openaiAudioTranscriptions'), observeProviderCall, http]), }; }; diff --git a/packages/provider-ollama/src/stages/prepare-request.ts b/packages/provider-ollama/src/stages/prepare-request.ts index 075563c12d..3aea95a1cd 100644 --- a/packages/provider-ollama/src/stages/prepare-request.ts +++ b/packages/provider-ollama/src/stages/prepare-request.ts @@ -1,16 +1,23 @@ import type { OllamaUpstreamConfig } from '../config.ts'; -import type { HttpRequestFacts, HttpResponseFacts } from '@floway-dev/http/pipeline'; +import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; import { multipartBody } from '@floway-dev/http/request-content'; import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { joinBaseAndPath, mergeHttpHeaders, withHttpContentType, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; -type OllamaOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiAudioTranscriptions'; +type OllamaOperation = 'openaiChatCompletions' | 'openaiResponses' | 'openaiResponsesCompact' | 'anthropicMessages' | 'anthropicMessagesCountTokens' | 'openaiCompletions' | 'openaiEmbeddings' | 'openaiAudioTranscriptions'; interface OllamaHttpFacts extends HttpRequestFacts { + 'request.provider.modelKey': string; 'request.ollama.modelKey': string; } const PATHS: Record = { + openaiChatCompletions: '/v1/chat/completions', + openaiResponses: '/v1/responses', + openaiResponsesCompact: '/v1/responses/compact', + anthropicMessages: '/v1/messages', + anthropicMessagesCountTokens: '/v1/messages/count_tokens', openaiCompletions: '/v1/completions', openaiEmbeddings: '/v1/embeddings', // https://github.com/ollama/ollama/blob/573386c35eac76124ffce571f4b0fefa0a7fe13c/server/routes.go#L1916-L1922 @@ -18,15 +25,15 @@ const PATHS: Record = { }; export const prepareOllamaRequest = (config: OllamaUpstreamConfig, operation: O) => { - const prepare = defineStage, OllamaHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ + const prepare = defineStage & { 'request.provider.anthropicBeta'?: readonly string[]; 'request.provider.responsesAction': 'generate' | 'compact' }, OllamaHttpFacts, ProviderResponse, ProviderResponse, ProviderServices>({ name: `prepareOllama${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, through: { request: { - needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + needs: [...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : [])], consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.ollama.modelKey'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.provider.modelKey', 'request.ollama.modelKey'], }, - response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next) => { const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; @@ -34,19 +41,27 @@ export const prepareOllamaRequest = (config: OllamaUp const encoding = operation === 'openaiAudioTranscriptions' ? 'multipart' as const : 'json' as const; const body = operation === 'openaiAudioTranscriptions' ? multipartBody((payload as ProviderOperationPayloads['openaiAudioTranscriptions']).entries.map(entry => entry.name === 'model' ? { ...entry, value: modelKey } : entry)) - : { ...payload, model: modelKey }; + : (operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact' + ? { ...toCompactPayloadShape(payload as ProviderOperationPayloads['openaiResponsesCompact']), model: modelKey } + : { ...payload, ...(operation === 'openaiChatCompletions' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' || operation === 'anthropicMessages' ? { stream: true } : {}), model: modelKey }; const base = config.apiKey ? [['Authorization', secret(`Bearer ${config.apiKey}`)] as const] : []; - const headers = mergeHttpHeaders(withHttpContentType(base, body, encoding), facts['request.http.headers']); + let headers = mergeHttpHeaders(withHttpContentType(base, body, encoding), facts['request.http.headers']); + if ('request.provider.anthropicBeta' in facts) { + const beta = facts['request.provider.anthropicBeta'] as readonly string[]; + headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); + if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; + } const back = await next(move({ ...rest, + 'request.provider.modelKey': modelKey, 'request.ollama.modelKey': modelKey, - 'request.http.url': joinBaseAndPath(config.baseUrl, PATHS[operation]), + 'request.http.url': joinBaseAndPath(config.baseUrl, ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') ? PATHS[facts['request.provider.responsesAction'] === 'compact' ? 'openaiResponsesCompact' : 'openaiResponses'] : PATHS[operation])), 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': body, 'request.http.encoding': encoding, })); - return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + return move({ ...back }); }, }); return prepare; diff --git a/packages/provider/__tests__/stages/decode-response_test.ts b/packages/provider/__tests__/stages/decode-response_test.ts new file mode 100644 index 0000000000..57e2c7a5ba --- /dev/null +++ b/packages/provider/__tests__/stages/decode-response_test.ts @@ -0,0 +1,69 @@ +import { expect, test } from 'vitest'; + +import type { ProviderChatResponse, ProviderChatServices } from '../../src/pipeline.ts'; +import { decodeProviderResponse } from '../../src/stages/decode-response.ts'; +import { observeProviderCall } from '../../src/stages/observe-call.ts'; +import { http, type HttpRequestFacts } from '@floway-dev/http/pipeline'; +import { compose, defineStage, getFailureFacts, move, run, setRelease } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; + +type Entry = HttpRequestFacts & { 'request.provider.modelKey': string }; +const entry = (): Entry => move({ + 'request.provider.modelKey': 'actual-wire-model', + 'request.http.callId': 1, + 'request.http.url': 'https://upstream.example/chat/completions', + 'request.http.method': 'POST', + 'request.http.headers': [], + 'request.http.body': { model: 'actual-wire-model', stream: true, messages: [] }, + 'request.http.encoding': 'json', +}); +const pipeline = compose>('provider.chat', [decodeProviderResponse('openaiChatCompletions'), observeProviderCall, http]); +const services = (response: () => Promise, record: ProviderChatServices['recordProtocolFrames'] = frames => frames): ProviderChatServices => ({ + httpCall: () => ({ fetcher: response, signal: undefined, waitUntil: () => {}, wrapUpstreamCall: call => call() }), + recordProtocolFrames: record, +}); + +test('stream decoding records its protocol stream and shares the single owned HTTP body', async () => { + let streams = 0; + const executed = await run(pipeline, entry(), services(async () => new Response('data: {"id":"reply","choices":[]}\n\ndata: [DONE]\n\n', { headers: { 'content-type': 'text/event-stream' } }), frames => { streams++; return frames; })); + const output = executed.facts['response.provider.output']; + expect(output).not.toBeNull(); + if (output === null || !('kind' in output) || output.kind !== 'stream') throw new Error('Expected decoded stream'); + const frames: ProtocolFrame[] = []; + for await (const frame of output.frames) frames.push(frame); + expect(frames.map(frame => frame.type)).toEqual(['event', 'done']); + expect(streams).toBe(1); + expect(executed.facts['response.provider.called']).toBe(true); + expect(executed.facts['response.provider.modelKey']).toBe('actual-wire-model'); + const exchange = executed.facts['response.http.exchange']; + if (exchange.type !== 'response' || exchange.body === null) throw new Error('Expected HTTP body'); + expect(executed.facts['response.http.body']).toBe(exchange.body); + setRelease(exchange.body, async () => {}); + await executed.drain(); +}); + +test('a successful non-SSE reply is a protocol failure value with its complete parsed payload', async () => { + const body = { error: { message: 'x'.repeat(4096) } }; + const executed = await run(pipeline, entry(), services(async () => Response.json(body))); + const output = executed.facts['response.provider.output']; + expect(output).toMatchObject({ status: 502, body }); + expect(executed.facts['response.provider.called']).toBe(true); + expect(executed.facts['response.http.exchange']).toMatchObject({ type: 'response', status: 200 }); + await executed.drain(); +}); + +test('actual HTTP replies remain failure facts when a higher decoder stage throws', async () => { + const error = new TypeError('programming error', { cause: new Error('original cause') }); + const fault = defineStage, ProviderChatResponse<'openaiChatCompletions'>>({ + name: 'faultAfterReply', + through: { request: { needs: [], consumes: [], provides: [] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => { await next(move({ ...facts })); throw error; }, + }); + try { + await run(compose>('provider.fault', [fault, decodeProviderResponse('openaiChatCompletions'), observeProviderCall, http]), entry(), services(async () => new Response('data: [DONE]\n\n', { headers: { 'content-type': 'text/event-stream' } }))); + throw new Error('Expected stage failure'); + } catch (caught) { + expect(caught).toBe(error); + expect(getFailureFacts(caught)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'actual-wire-model', 'response.provider.previousCalls': [] }); + } +}); diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index 2ffdd520d8..e6eb0ddad8 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -7,8 +7,11 @@ export type { OpenAIResponsesInvocation, } from './invocation.ts'; export { providerModelOf } from './invocation.ts'; -export type { ProviderCallResponse, ProviderModelFacts, ProviderOperation, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderPipelines, ProviderRequest, ProviderResponse, ProviderRerankResponse, ProviderServices } from './pipeline.ts'; +export type { AnthropicMessagesCountTokensBody, ChatProviderOperation, NonChatProviderOperation, ProviderCallResponse, ProviderChatResponse, ProviderChatServices, ProviderModelFacts, ProviderOperation, ProviderOperationOutputs, ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipeline, ProviderPipelines, ProviderProtocolFailure, ProviderRequest, ProviderResponse, ProviderRerankResponse, ProviderServices, ProviderStreamOutput, ProviderValueOutput } from './pipeline.ts'; export { providerModelFacts } from './pipeline.ts'; +export { decodeProviderResponse } from './stages/decode-response.ts'; +export { selectProviderResponsesAction } from './stages/select-responses-action.ts'; +export { observeProviderCall } from './stages/observe-call.ts'; export { mergeHttpHeaders, replaceHttpHeader, withHttpContentType } from './http-headers.ts'; export type { InternalDebugError } from './error.ts'; diff --git a/packages/provider/src/pipeline.ts b/packages/provider/src/pipeline.ts index 2dfaaedee9..14c3971fd1 100644 --- a/packages/provider/src/pipeline.ts +++ b/packages/provider/src/pipeline.ts @@ -4,13 +4,21 @@ import type { OpenAIImagesEditsRequest } from './images.ts'; import type { ProviderModel } from './model.ts'; import type { HttpResponseFacts, HttpHeaders, HttpServices } from '@floway-dev/http/pipeline'; import type { Pipeline } from '@floway-dev/pipeline'; -import type { RerankTarget } from '@floway-dev/protocols/common'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { OpenAICompletionsPayload } from '@floway-dev/protocols/openai-completions'; import type { OpenAIEmbeddingsPayload } from '@floway-dev/protocols/openai-embeddings'; import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; export interface ProviderOperationPayloads { + openaiChatCompletions: Omit; + openaiResponses: Omit; + openaiResponsesCompact: Omit; + anthropicMessages: Omit; + anthropicMessagesCountTokens: Omit; alphaSearch: Record; openaiCompletions: Omit; openaiEmbeddings: Omit; @@ -21,6 +29,8 @@ export interface ProviderOperationPayloads { } export type ProviderOperation = keyof ProviderOperationPayloads; +export type ChatProviderOperation = 'openaiChatCompletions' | 'openaiResponses' | 'openaiResponsesCompact' | 'anthropicMessages' | 'anthropicMessagesCountTokens'; +export type NonChatProviderOperation = Exclude; export type ProviderModelFacts = Omit & { readonly enabledFlags: readonly FlagId[] }; @@ -33,6 +43,9 @@ export interface ProviderRequest

{ 'request.http.headers': HttpHeaders; } +export type ProviderOperationRequest = ProviderRequest & + (O extends 'anthropicMessages' | 'anthropicMessagesCountTokens' ? { 'request.provider.anthropicBeta': readonly string[] } : object); + export interface ProviderCallResponse extends HttpResponseFacts { /** A preflight reply does not represent a call to the model endpoint. */ 'response.provider.called': boolean; @@ -48,13 +61,42 @@ export interface ProviderRerankResponse extends ProviderResponse { 'response.provider.rerankTarget': RerankTarget; } +export interface ProviderProtocolFailure { + readonly status: number; + readonly message: string; + readonly body?: unknown; +} + +export type ProviderStreamOutput = { readonly kind: 'stream'; readonly frames: AsyncIterable> }; +export type ProviderValueOutput = { readonly kind: 'value'; readonly body: T }; +export interface AnthropicMessagesCountTokensBody { + readonly input_tokens: number; + readonly [key: string]: unknown; +} + +export interface ProviderOperationOutputs { + openaiChatCompletions: ProviderStreamOutput; + openaiResponses: ProviderStreamOutput | ProviderValueOutput; + openaiResponsesCompact: ProviderStreamOutput | ProviderValueOutput; + anthropicMessages: ProviderStreamOutput; + anthropicMessagesCountTokens: ProviderValueOutput; +} + +export interface ProviderChatResponse extends ProviderResponse { + 'response.provider.output': ProviderOperationOutputs[O] | ProviderProtocolFailure | null; +} + export type ProviderOperationResponse = - O extends 'rerank' ? ProviderRerankResponse : ProviderResponse; + O extends ChatProviderOperation ? ProviderChatResponse : O extends 'rerank' ? ProviderRerankResponse : ProviderResponse; export type ProviderPipeline = Pipeline< - ProviderRequest, + ProviderOperationRequest, ProviderOperationResponse >; export type ProviderPipelines = { readonly [O in ProviderOperation]?: ProviderPipeline }; export type ProviderServices = HttpServices; + +export type ProviderChatServices = ProviderServices & { + readonly recordProtocolFrames: >(frames: AsyncIterable) => AsyncIterable; +}; diff --git a/packages/provider/src/stages/decode-response.ts b/packages/provider/src/stages/decode-response.ts new file mode 100644 index 0000000000..461833af83 --- /dev/null +++ b/packages/provider/src/stages/decode-response.ts @@ -0,0 +1,58 @@ +import type { ChatProviderOperation, ProviderChatResponse, ProviderChatServices, ProviderOperationOutputs, ProviderProtocolFailure } from '../pipeline.ts'; +import { exchangeResponse, type HttpRequestFacts, type HttpResponseFacts, type HttpResponseExchange } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease } from '@floway-dev/pipeline'; +import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; +import { isEventStreamMediaType } from '@floway-dev/protocols/common'; +import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; +import { parseOpenAIResponsesStream } from '@floway-dev/protocols/openai-responses'; + +type Request = Pick & { 'request.provider.responsesAction': 'generate' | 'compact' }; +type Answer = HttpResponseFacts & Pick, 'response.provider.output'>; + +const parsedBody = (text: string): unknown => { + try { return JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; return text; } +}; + +const protocolFailure = (exchange: HttpResponseExchange, text: string): ProviderProtocolFailure => { + const type = exchange.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''; + const snippet = text.length === 0 ? '' : text.length > 1024 ? `${text.slice(0, 1024)}...[truncated]` : text; + return { + status: 502, + message: `Upstream returned ${exchange.status} with content-type "${type || 'unknown'}" but stream is required (provider must force stream=true and return text/event-stream when response.ok). Body: ${snippet}`, + body: parsedBody(text), + }; +}; + +export const decodeProviderResponse = (operation: O) => defineStage, ProviderChatServices>({ + name: `decodeProvider${operation.replace(/^openai/, 'OpenAI').replace(/^anthropic/, 'Anthropic')}`, + through: { + request: { needs: ['request.http.callId', ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : [])], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.body'], provides: ['response.http.body', 'response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + const output = (answer: ProviderOperationOutputs[O] | ProviderProtocolFailure | null) => move({ ...back, 'response.provider.output': answer }); + if (exchange.type === 'transportFailure' || exchange.status < 200 || exchange.status >= 300) return output(null); + if ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact' || operation === 'anthropicMessagesCountTokens') { + const text = await exchangeResponse(exchange).text(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + let body: unknown; + try { body = JSON.parse(text); } catch (error) { + if (!(error instanceof SyntaxError)) throw error; + return output({ status: 502, message: `Upstream returned invalid ${operation} JSON`, body: text }); + } + return output({ kind: 'value', body } as ProviderOperationOutputs[O]); + } + const contentType = exchange.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1]; + if (exchange.body === null || !isEventStreamMediaType(contentType)) { + const text = await exchangeResponse(exchange).text(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + return output(protocolFailure(exchange, text)); + } + const signal = use.httpCall(facts['request.http.callId']).signal; + if (operation === 'anthropicMessages') return output({ kind: 'stream', frames: use.recordProtocolFrames(parseAnthropicMessagesStream(exchange.body, { signal })) } as ProviderOperationOutputs[O]); + if (operation === 'openaiChatCompletions') return output({ kind: 'stream', frames: use.recordProtocolFrames(parseOpenAIChatCompletionsStream(exchange.body, { signal })) } as ProviderOperationOutputs[O]); + return output({ kind: 'stream', frames: use.recordProtocolFrames(parseOpenAIResponsesStream(exchange.body, { signal })) } as ProviderOperationOutputs[O]); + }, +}); diff --git a/packages/provider/src/stages/observe-call.ts b/packages/provider/src/stages/observe-call.ts new file mode 100644 index 0000000000..d2f970a5a9 --- /dev/null +++ b/packages/provider/src/stages/observe-call.ts @@ -0,0 +1,22 @@ +import type { ProviderResponse, ProviderServices } from '../pipeline.ts'; +import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defineStage, move } from '@floway-dev/pipeline'; + +type Request = { 'request.provider.modelKey': string }; + +export const observeProviderCall = defineStage({ + name: 'observeProviderCall', + through: { + request: { needs: ['request.provider.modelKey'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + }, + execute: async (facts, next) => { + const back = await next(move({ ...facts })); + return move({ + ...back, + 'response.provider.modelKey': facts['request.provider.modelKey'], + 'response.provider.called': back['response.http.exchange'].type === 'response', + 'response.provider.previousCalls': [], + }); + }, +}); diff --git a/packages/provider/src/stages/select-responses-action.ts b/packages/provider/src/stages/select-responses-action.ts new file mode 100644 index 0000000000..2fcf72bd2b --- /dev/null +++ b/packages/provider/src/stages/select-responses-action.ts @@ -0,0 +1,12 @@ +import { defineStage, move } from '@floway-dev/pipeline'; + +type Facts = { 'request.provider.responsesAction': 'generate' | 'compact' }; + +export const selectProviderResponsesAction = (action: Facts['request.provider.responsesAction']) => defineStage({ + name: 'selectProviderResponsesAction', + through: { + request: { needs: [], consumes: [], provides: ['request.provider.responsesAction'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.responsesAction': action })) }), +}); diff --git a/packages/test-utils/src/provider-pipeline.ts b/packages/test-utils/src/provider-pipeline.ts index b6dae79a9f..2cb2ea9102 100644 --- a/packages/test-utils/src/provider-pipeline.ts +++ b/packages/test-utils/src/provider-pipeline.ts @@ -1,13 +1,13 @@ import { noopUpstreamCallOptions } from './stubs.ts'; import { exchangeResponse, type HttpServices } from '@floway-dev/http/pipeline'; import { move, run, setRelease } from '@floway-dev/pipeline'; -import type { Provider, ProviderCallResult, ProviderModel, ProviderOperation, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderRerankCallResult, ProviderRequest, UpstreamCallOptions } from '@floway-dev/provider'; +import type { NonChatProviderOperation, Provider, ProviderCallResult, ProviderModel, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderRerankCallResult, ProviderOperationRequest, UpstreamCallOptions } from '@floway-dev/provider'; import { providerModelFacts } from '@floway-dev/provider'; -type CallResult = +type CallResult = (O extends 'rerank' ? ProviderRerankCallResult : ProviderCallResult) & { readonly called: boolean; readonly previousCalls: readonly { readonly modelKey: string }[] }; -export const callProviderPipeline = async ( +export const callProviderPipeline = async ( provider: Provider, operation: O, model: ProviderModel, @@ -17,14 +17,14 @@ export const callProviderPipeline = async ( ): Promise> => { const pipeline = provider.pipelines[operation]; if (pipeline === undefined) throw new Error(`${provider.kind} has no ${operation} pipeline`); - const entry: ProviderRequest = move({ + const entry = move({ 'request.provider.model': providerModelFacts(model), 'request.provider.payload': payload, 'request.http.callId': 0, 'request.http.headers': [...options.headers], - }); + }) as ProviderOperationRequest; const services: HttpServices = { httpCall: () => ({ ...options, signal }) }; - const executed = await run, ProviderOperationResponse, HttpServices>(pipeline as ProviderPipeline, entry, services); + const executed = await run, ProviderOperationResponse, HttpServices>(pipeline as ProviderPipeline, entry, services); const facts = executed.facts; const exchange = facts['response.http.exchange']; if (exchange.type === 'transportFailure') { From 9c17d5079ec539b3660d1e4a702752834174d269 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 05:17:07 +0800 Subject: [PATCH 118/156] fix: record protocol errors for abnormal client streams Complete source and serialization failures within the owning protocol before recording and transport framing. Preserve original debug causes, join client failure with measured upstream usage, and keep clean Chat Completions EOF valid. Verified 78 focused chat and client-wire tests, including exact dump/SSE parity for source and renderer failures. --- .../chat/anthropic-messages-pipeline_test.ts | 4 +- .../data-plane/chat/client-errors_test.ts | 102 ++++++++++++++++++ .../data-plane/chat/client-streams_test.ts | 4 +- .../gemini-generate-content-pipeline_test.ts | 3 +- .../chat/openai-responses/emit_test.ts | 9 +- .../chat/anthropic-messages/emit.ts | 33 +++--- .../anthropic-messages/web-search-tool.ts | 10 +- .../chat/gemini-generate-content/emit.ts | 37 ++++--- .../chat/openai-chat-completions/emit.ts | 44 ++++---- .../data-plane/chat/openai-responses/emit.ts | 72 ++++++------- .../data-plane/chat/shared/client-stream.ts | 49 +++++++++ .../src/openai-chat-completions/index.ts | 6 ++ .../src/openai-chat-completions/to-sse.ts | 8 +- 13 files changed, 275 insertions(+), 106 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/client-errors_test.ts create mode 100644 packages/gateway/src/data-plane/chat/shared/client-stream.ts diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts index a9d114ff14..85bb89f251 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts @@ -392,6 +392,8 @@ describe('the messages chain', () => { const { facts } = await serve(true); - await expect(collect(facts['response.chat.anthropicMessages.rendered'])).rejects.toThrow(ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE); + const frames = await collect(facts['response.chat.anthropicMessages.rendered']); + expect(frames.at(-1)?.event).toBe('error'); + expect(JSON.parse(frames.at(-1)!.data)).toMatchObject({ type: 'error', error: { message: ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE } }); }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/client-errors_test.ts b/packages/gateway/__tests__/data-plane/chat/client-errors_test.ts new file mode 100644 index 0000000000..7e2a6a8244 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/client-errors_test.ts @@ -0,0 +1,102 @@ +import { Hono } from 'hono'; +import { expect, test, vi } from 'vitest'; + +import { emitAnthropicMessages } from '../../../src/data-plane/chat/anthropic-messages/emit.ts'; +import { meterUsage as meterAnthropic } from '../../../src/data-plane/chat/anthropic-messages/meter.ts'; +import { emitGeminiGenerateContent } from '../../../src/data-plane/chat/gemini-generate-content/emit.ts'; +import { emitOpenAIChatCompletions } from '../../../src/data-plane/chat/openai-chat-completions/emit.ts'; +import { meterUsage as meterChat } from '../../../src/data-plane/chat/openai-chat-completions/meter.ts'; +import { prologueFor, serveThrough } from '../../../src/data-plane/pipeline/serve.ts'; +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { openRunDump } from '../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import type { ApiKey } from '../../../src/repo/types.ts'; +import { eventsOf, installDumpStubs } from '../../dump/test-fixtures.ts'; +import { InMemoryRepo } from '../../repo/memory.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, createRunReader, defineStage, move, defer, type Deferred, type DumpEvent } from '@floway-dev/pipeline'; +import { eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { testTelemetryModelIdentity } from '@floway-dev/test-utils'; + +const key: ApiKey = { id: 'client-key', userId: 1, name: 'Client key', key: 'client-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; +const fixtures = [ + { protocol: 'openaiChatCompletions', emit: emitOpenAIChatCompletions, meter: meterChat, initial: { id: 'chat', object: 'chat.completion.chunk', created: 1, model: 'model', choices: [{ index: 0, delta: { content: 'partial' }, finish_reason: null }] } }, + { protocol: 'anthropicMessages', emit: emitAnthropicMessages, meter: meterAnthropic, initial: { type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'partial' } } }, + { protocol: 'geminiGenerateContent', emit: emitGeminiGenerateContent, meter: null, initial: { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'partial' }] } }] } }, +]; +type Exit = Record & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] }; + +for (const fixture of fixtures) for (const failureAt of ['read', 'render'] as const) { + test(`actual ${fixture.protocol} SSE records and exposes ${failureAt} failure with its original error chain`, async () => { + initRepo(new InMemoryRepo()); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const pending: Promise[] = []; + const schedule = (work: Promise) => { pending.push(work); }; + const dump = openRunDump(key, { method: 'POST', path: '/v1/chat', body: { bytes: new Uint8Array(), streamError: null } }, schedule, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + if (dump === null) throw new Error('retained request has no recording'); + const gateway = mockChatGatewayCtx({ apiKeyId: key.id, dump, wantsStream: true, backgroundScheduler: schedule }); + const fault = new Error('stream socket broke', { cause: new Error('socket reset') }); + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}); + let completed = false; + let settle!: (outcome: { billable: readonly never[]; failed: boolean }) => void; + const reading = defer(new Promise<{ billable: readonly never[]; failed: boolean }>(resolve => { settle = resolve; })); + const responseKey = `response.chat.${fixture.protocol}`; + const usageKey = `${responseKey}.streamedUsage`; + const dial = defineStage, Record>({ + name: 'scriptedDial', return: { provides: [responseKey, usageKey, 'response.http.headers', 'response.http.status', 'response.usage.billable'] }, + execute: async facts => move({ + ...facts, 'response.http.headers': [], 'response.http.status': 200, + 'response.usage.billable': [{ identity: testTelemetryModelIdentity, quantities: {} }], + [usageKey]: fixture.meter === null ? reading : null, + [responseKey]: { + kind: 'stream', frames: { + async *[Symbol.asyncIterator]() { + try { + yield eventFrame({ ...fixture.initial, ...(failureAt === 'render' ? { impossibleJson: 1n } : {}) }); + if (failureAt === 'read') throw fault; + } finally { + completed = true; + settle({ billable: [], failed: true }); + } + }, + }, + }, + }), + }); + const pipeline = compose, Exit>('clientError', [fixture.emit, ...(fixture.meter === null ? [] : [fixture.meter(usageKey)]), dial]); + const app = new Hono(); + app.post('/v1/chat', c => serveThrough(c, prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }, dump), pipeline, + move({ [`ingress.chat.${fixture.protocol}.wantsStream`]: true, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false }), + facts => ({ frames: facts[`${responseKey}.rendered`] as AsyncIterable }), + facts => facts[usageKey] as Deferred<{ billable: readonly never[]; failed: boolean }>)); + const response = await app.request('/v1/chat', { method: 'POST' }); + const body = await response.text(); + await Promise.all(pending); + expect(completed).toBe(true); + expect(logged).not.toHaveBeenCalled(); + logged.mockRestore(); + const actual = body.split('\n').filter(line => line.startsWith('data: ')).map(line => JSON.parse(line.slice(6)) as { error?: { message?: string; cause?: { message: string } } }); + const error = actual.find(event => event.error !== undefined)?.error; + expect(error?.message).toBe(failureAt === 'read' ? fault.message : 'Do not know how to serialize a BigInt'); + if (failureAt === 'read') expect(error?.cause?.message).toBe('socket reset'); + expect(body).not.toContain('[DONE]'); + expect(body).not.toContain('impossibleJson'); + const record = stubs.stored[0]!.record; + const read = createRunReader(); + const streams = new Map(); + let client: number | null = null; + for (const item of eventsOf(record)) { + const event = item as unknown as DumpEvent; + const decoded = read(event); + if (decoded?.facts && 'response.chat.clientFrames' in decoded.facts) client = (decoded.facts['response.chat.clientFrames'] as { stream: number }).stream; + if (event.type === 'stream.frame') { + const frames = streams.get(event.streamId) ?? []; + frames.push(...(decoded?.frames as ProtocolFrame[]).map(frame => frame.type === 'event' ? frame.event : '[DONE]')); + streams.set(event.streamId, frames); + } + } + expect(client).not.toBeNull(); + expect(streams.get(client!)).toEqual(actual); + expect(record.meta.error).toMatchObject({ kind: 'failed' }); + }); +} diff --git a/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts b/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts index 2920ec6de7..476f65f50f 100644 --- a/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts @@ -64,9 +64,9 @@ for (const fixture of cases) { const dump = openRunDump(key, { method: 'POST', path: '/v1/chat', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); if (dump === null) throw new Error('retained client request has no run recording'); const dial = defineStage, Record>({ - name: 'scriptedDial', return: { provides: [fixture.response, 'response.http.headers'] }, + name: 'scriptedDial', return: { provides: [fixture.response, 'response.http.headers', `${fixture.response}.streamedUsage`] }, execute: async facts => move({ - ...facts, 'response.http.headers': [], + ...facts, 'response.http.headers': [], [`${fixture.response}.streamedUsage`]: null, [fixture.response]: { kind: 'stream', frames: { async *[Symbol.asyncIterator]() { yield* fixture.upstream; } } }, }), }); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts index 3a34e0178a..c2a40f97f4 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts @@ -402,7 +402,8 @@ describe('the Gemini generateContent pipeline', () => { const { facts, drain } = await serve(entryFacts()); - await expect(collect(facts['response.chat.geminiGenerateContent.rendered'])).rejects.toThrow(GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE); + const frames = await collect(facts['response.chat.geminiGenerateContent.rendered']); + expect(JSON.parse(frames.at(-1)!.data)).toMatchObject({ error: { code: 500, status: 'INTERNAL', message: GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE } }); await drain(); }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts index f35d10f834..19749c3b59 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts @@ -31,7 +31,7 @@ beforeEach(() => { }); afterEach(() => { vi.restoreAllMocks(); }); -for (const failureAt of ['before-response', 'upstream-stream', 'snapshot'] as const) { +for (const failureAt of ['before-response', 'upstream-stream', 'snapshot', 'render'] as const) { test(`records the exact client protocol failure frames for ${failureAt} failure`, async () => { const announced = failureAt !== 'before-response'; const stubs = installDumpStubs(initDumpStore, initDumpBroker); @@ -57,6 +57,10 @@ for (const failureAt of ['before-response', 'upstream-stream', 'snapshot'] as co }, }; } + if (failureAt === 'render') { + yield { type: 'event', event: { type: 'response.in_progress', response: { id: 'upstream-response', object: 'response', model: 'model', status: 'in_progress', output: [], error: null, incomplete_details: null, impossibleJson: 1n } } }; + return; + } if (failureAt === 'snapshot') { yield { type: 'event', event: { @@ -116,7 +120,8 @@ for (const failureAt of ['before-response', 'upstream-stream', 'snapshot'] as co : announced ? ['response.created', 'error', 'response.failed'] : ['error']; expect(frames.filter(frame => frame.type === 'event').map(frame => frame.event.type)).toEqual(expectedEvents); const error = frames.find(frame => frame.type === 'event' && frame.event.type === 'error'); - expect(error).toMatchObject({ event: { error: { message: fault.message, cause: { message: 'socket reset' } } } }); + expect(error).toMatchObject({ event: { error: { message: failureAt === 'render' ? 'Do not know how to serialize a BigInt' : fault.message } } }); + if (failureAt !== 'render') expect(error).toMatchObject({ event: { error: { cause: { message: 'socket reset' } } } }); if (announced) { const created = frames[0] as ProtocolFrame>; if (created.type !== 'event') throw new Error('announced response has no creation event'); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts index f3a8f1ee0a..9e72377991 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts @@ -1,14 +1,16 @@ import { wrapAnthropicMessagesAffinityEgress } from './affinity/egress.ts'; import { renderAnthropicMessagesError } from './errors.ts'; import type { Fields } from './facts.ts'; -import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; +import { recordStream } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure, mintedAs } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; +import { framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { collectAnthropicMessagesProtocolEventsToResult, anthropicMessagesEventToSsePayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; +import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; +import { toInternalDebugError } from '@floway-dev/provider'; /** * The outermost edge. An Anthropic Messages answer is always a stream by the time it reaches here — @@ -23,17 +25,17 @@ import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame, type SseFram export const emitAnthropicMessages = defineStage< Fields<'ingress.chat.anthropicMessages.wantsStream'>, Fields<'ingress.chat.anthropicMessages.wantsStream'>, - Fields<'ingress.chat.anthropicMessages.wantsStream' | 'response.chat.anthropicMessages' | 'response.http.headers'>, - Fields<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames'>, + Fields<'ingress.chat.anthropicMessages.wantsStream' | 'response.chat.anthropicMessages' | 'response.http.headers' | 'response.chat.anthropicMessages.streamedUsage'>, + Fields<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.anthropicMessages.streamedUsage'>, ChatServices >({ name: 'emitAnthropicMessages', through: { request: { needs: ['ingress.chat.anthropicMessages.wantsStream'], consumes: [], provides: [] }, response: { - needs: ['response.chat.anthropicMessages', 'response.http.headers'], + needs: ['response.chat.anthropicMessages', 'response.http.headers', 'response.chat.anthropicMessages.streamedUsage'], consumes: ['response.chat.anthropicMessages', 'response.http.headers'], - provides: ['response.chat.clientFrames', 'response.chat.anthropicMessages.rendered', 'response.http.status', 'response.http.headers'], + provides: ['response.chat.clientFrames', 'response.chat.anthropicMessages.rendered', 'response.http.status', 'response.http.headers', 'response.chat.anthropicMessages.streamedUsage'], }, }, execute: async (facts, next, use) => { @@ -85,12 +87,18 @@ export const emitAnthropicMessages = defineStage< 'response.http.status': 200, }; } - const frames = recordStream(clientFrames(egress), use.gateway.dump); + const completed = framedClientStream( + clientFrames(egress), + frame => sseFrame(JSON.stringify(frame.event), frame.event.type), + error => [eventFrame({ type: 'error' as const, error: toInternalDebugError(error, 'anthropicMessages') })], + use.gateway.dump, + ); return { ...rest, - 'response.chat.clientFrames': move(frames), + 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, - 'response.chat.anthropicMessages.rendered': move(renderSSE(frames)), + 'response.chat.anthropicMessages.rendered': move(completed.rendered), + 'response.chat.anthropicMessages.streamedUsage': move(withClientVerdict(back['response.chat.anthropicMessages.streamedUsage'], completed.failed)), 'response.http.status': 200, }; }, @@ -108,13 +116,6 @@ const clientFrames = (frames: AsyncIterable>): AsyncIterable => ({ - ...streamReferenceOf(frames), - [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) yield sseFrame(JSON.stringify(frame.event), frame.event.type); - })(), -}); - /** What this protocol writes on an idle connection. Anthropic defines a `ping` event and * clients read it, so an SSE comment — invisible by design — is not the same wire byte. * The route that serves this chain hands it to the seam alongside the frames. */ diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts index 1aa69b3dc2..6cff05cd0d 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts @@ -6,10 +6,11 @@ // by this gateway, and the answer rewritten back into Anthropic's own // `server_tool_use` / `web_search_tool_result` shape on the way up. // -// One descent, unlike the OpenAI Responses shim: Anthropic's protocol carries the tool result +// One descent, unlike the OpenAI Responses hosted-tool loop: Anthropic's protocol carries the tool result // inside the same turn, so there is nothing to ask again for. What this stage is, then, is a // request rewrite and a response rewrite around one dial. +import type { Fields } from './facts.ts'; import { anthropicMessagesWebSearchInvalidRequestBody, prepareAnthropicMessagesWebSearchInvocation, @@ -27,10 +28,10 @@ import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; type M = { [P in K]: ChatFacts[P] }; -type Answered = M<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers'> & Record; +type Answered = Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers' | 'response.chat.anthropicMessages.streamedUsage'> & Record; /** The gateway's own refusal, in Anthropic's words. It is written here rather than dialled, - * because a tool declaration this shim cannot execute would reach the upstream as a body this + * because a tool declaration this gateway cannot execute would reach the upstream as a body this * gateway wrote. The envelope is stated as itself: a refusal this gateway authored has no * upstream bytes behind it, so there is nothing to serialize and read back. */ const refusal = (message: string): Failure => ({ @@ -69,7 +70,7 @@ export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => de // A declaration this gateway cannot execute, and a search backend the operator has not // configured, are both answered here: neither is a body an upstream should be asked about. return: { - provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers'], + provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.chat.anthropicMessages.streamedUsage'], }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -89,6 +90,7 @@ export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => de return move({ ...facts, 'response.chat.anthropicMessages': refusal(prepared.message), + 'response.chat.anthropicMessages.streamedUsage': null, 'response.usage.billable': [], 'response.http.headers': [], }) as never; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts index 75cc2ec8d4..0d9ffed020 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts @@ -1,14 +1,16 @@ import { wrapGeminiGenerateContentAffinityEgress } from './affinity/egress.ts'; import { mintGeminiGenerateContentFailure } from './errors.ts'; import type { Fields } from './facts.ts'; -import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; +import { recordStream } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; +import { framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import { sseFrame, type EventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectGeminiGenerateContentProtocolEventsToResult, type GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; +import { toInternalDebugError } from '@floway-dev/provider'; /** * The outermost edge. A Gemini generateContent answer is always a stream by the time it reaches @@ -22,8 +24,8 @@ import { collectGeminiGenerateContentProtocolEventsToResult, type GeminiGenerate export const emitGeminiGenerateContent = defineStage< Fields<'ingress.chat.geminiGenerateContent.wantsStream'>, Fields<'ingress.chat.geminiGenerateContent.wantsStream'>, - Fields<'ingress.chat.geminiGenerateContent.wantsStream' | 'response.chat.geminiGenerateContent' | 'response.http.headers'>, - Fields<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames'>, + Fields<'ingress.chat.geminiGenerateContent.wantsStream' | 'response.chat.geminiGenerateContent' | 'response.http.headers' | 'response.chat.geminiGenerateContent.streamedUsage'>, + Fields<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.geminiGenerateContent.streamedUsage'>, ChatServices >({ name: 'emitGeminiGenerateContent', @@ -34,9 +36,9 @@ export const emitGeminiGenerateContent = defineStage< provides: [], }, response: { - needs: ['response.chat.geminiGenerateContent', 'response.http.headers'], + needs: ['response.chat.geminiGenerateContent', 'response.http.headers', 'response.chat.geminiGenerateContent.streamedUsage'], consumes: ['response.chat.geminiGenerateContent', 'response.http.headers'], - provides: ['response.chat.clientFrames', 'response.chat.geminiGenerateContent.rendered', 'response.http.status', 'response.http.headers'], + provides: ['response.chat.clientFrames', 'response.chat.geminiGenerateContent.rendered', 'response.http.status', 'response.http.headers', 'response.chat.geminiGenerateContent.streamedUsage'], }, }, execute: async (facts, next, use) => { @@ -88,12 +90,18 @@ export const emitGeminiGenerateContent = defineStage< 'response.http.status': 200, }; } - const frames = recordStream(clientFrames(egress), use.gateway.dump); + const completed = framedClientStream( + clientFrames(egress), + frame => sseFrame(JSON.stringify(frame.event)), + error => [eventFrame(geminiStreamError(error))], + use.gateway.dump, + ); return { ...rest, - 'response.chat.clientFrames': move(frames), + 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, - 'response.chat.geminiGenerateContent.rendered': move(renderSSE(frames)), + 'response.chat.geminiGenerateContent.rendered': move(completed.rendered), + 'response.chat.geminiGenerateContent.streamedUsage': move(withClientVerdict(back['response.chat.geminiGenerateContent.streamedUsage'], completed.failed)), 'response.http.status': 200, }; }, @@ -105,9 +113,8 @@ const clientFrames = (frames: AsyncIterable>): AsyncIterable => ({ - ...streamReferenceOf(frames), - [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) yield sseFrame(JSON.stringify(frame.event)); - })(), -}); +const geminiStreamError = (error: unknown): GeminiGenerateContentStreamEvent => { + const debug = toInternalDebugError(error, 'geminiGenerateContent'); + const envelope = mintGeminiGenerateContentFailure({ status: 500, message: debug.message }).body; + return { error: { ...(envelope.error as { code: number; message: string; status: string }), ...debug } }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts index 2c6875d47e..6665b775c2 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts @@ -1,13 +1,15 @@ import { wrapOpenAIChatCompletionsAffinityEgress } from './affinity/egress.ts'; import type { Fields } from './facts.ts'; -import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; +import { recordStream } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; +import { framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; -import { collectOpenAIChatCompletionsProtocolEventsToResult, openaiChatCompletionsProtocolFrameToSSEFrame, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import { eventFrame, isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { collectOpenAIChatCompletionsProtocolEventsToResult, openaiChatCompletionsProtocolFrameToSSEFrame, type OpenAIChatCompletionsStreamEvent, type ClientOpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import { toInternalDebugError } from '@floway-dev/provider'; /** * The outermost edge. A chat answer is always a stream by the time it reaches here — the @@ -21,8 +23,8 @@ export const emitOpenAIChatCompletions = defineStage< Fields<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, Fields<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, Fields<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk' - | 'response.chat.openaiChatCompletions' | 'response.http.headers'>, - Fields<'response.chat.openaiChatCompletions.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames'>, + | 'response.chat.openaiChatCompletions' | 'response.http.headers' | 'response.chat.openaiChatCompletions.streamedUsage'>, + Fields<'response.chat.openaiChatCompletions.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.openaiChatCompletions.streamedUsage'>, ChatServices >({ name: 'emitOpenAIChatCompletions', @@ -33,9 +35,9 @@ export const emitOpenAIChatCompletions = defineStage< provides: [], }, response: { - needs: ['response.chat.openaiChatCompletions', 'response.http.headers'], + needs: ['response.chat.openaiChatCompletions', 'response.http.headers', 'response.chat.openaiChatCompletions.streamedUsage'], consumes: ['response.chat.openaiChatCompletions', 'response.http.headers'], - provides: ['response.chat.clientFrames', 'response.chat.openaiChatCompletions.rendered', 'response.http.status', 'response.http.headers'], + provides: ['response.chat.clientFrames', 'response.chat.openaiChatCompletions.rendered', 'response.http.status', 'response.http.headers', 'response.chat.openaiChatCompletions.streamedUsage'], }, }, execute: async (facts, next, use) => { @@ -75,12 +77,8 @@ export const emitOpenAIChatCompletions = defineStage< answer.frames as AsyncIterable>, affinityEgressOptions(use.gateway), ); - const frames = recordStream( - back['ingress.chat.openaiChatCompletions.wantsStream'] - ? clientFrames(egress, back['ingress.chat.openaiChatCompletions.wantsUsageChunk']) : egress, - use.gateway.dump, - ); if (!back['ingress.chat.openaiChatCompletions.wantsStream']) { + const frames = recordStream(egress, use.gateway.dump); return { ...rest, 'response.chat.clientFrames': move(frames), @@ -91,11 +89,18 @@ export const emitOpenAIChatCompletions = defineStage< 'response.http.status': 200, }; } + const completed = framedClientStream>( + clientFrames(egress, back['ingress.chat.openaiChatCompletions.wantsUsageChunk']), + frame => openaiChatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk: true }), + error => [eventFrame({ error: toInternalDebugError(error, 'openaiChatCompletions') })], + use.gateway.dump, + ); return { ...rest, - 'response.chat.clientFrames': move(frames), + 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, - 'response.chat.openaiChatCompletions.rendered': move(renderSSE(frames)), + 'response.chat.openaiChatCompletions.rendered': move(completed.rendered), + 'response.chat.openaiChatCompletions.streamedUsage': move(withClientVerdict(back['response.chat.openaiChatCompletions.streamedUsage'], completed.failed)), 'response.http.status': 200, }; }, @@ -111,14 +116,3 @@ const clientFrames = (frames: AsyncIterable>, -): AsyncIterable => ({ - // The frames the client reads are a reframing of the ones the record holds, so this key - // points at that same stream rather than at nothing. - ...streamReferenceOf(frames), - [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) yield openaiChatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk: true }); - })(), -}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts index a1efca4723..0aa2f1310f 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts @@ -1,13 +1,13 @@ import { wrapOpenAIResponsesClientEgress } from './client-output.ts'; import { internalErrorEnvelope } from './errors.ts'; import { OPENAI_RESPONSES_STREAMED_USAGE, type OpenAIResponsesStreamFraming, type Fields } from './facts.ts'; -import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; +import { recordStream } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/facts.ts'; -import type { StreamOutcome } from '../../pipeline/serve.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; -import { defer, defineStage, move, type Deferred } from '@floway-dev/pipeline'; -import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectOpenAIResponsesProtocolEventsToResult, openaiResponsesProtocolFrameToSSEFrame, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent, type ClientOpenAIResponsesStreamEvent, type ClientResponseResource } from '@floway-dev/protocols/openai-responses'; import { toInternalDebugError } from '@floway-dev/provider'; @@ -124,15 +124,36 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra }; } } - const clientStream = completeStream(egress, framing, error => { use.gateway.dump?.failed(error); }); - const frames = recordStream(clientStream.frames, use.gateway.dump); + if (framing === 'events') { + const clientStream = completeEventStream(egress, error => { use.gateway.dump?.failed(error); }); + const frames = recordStream(clientStream.frames, use.gateway.dump); + return { + ...rest, + 'response.chat.clientFrames': move(frames), + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move(frames), + 'response.http.status': 200, + [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], clientStream.failed)), + }; + } + let announced: ClientResponseResource | undefined; + const completed = framedClientStream( + completedSseFrames(egress), + frame => { + const wire = openaiResponsesProtocolFrameToSSEFrame(frame); + if (frame.type === 'event' && 'response' in frame.event) announced = frame.event.response; + return wire; + }, + error => [eventFrame(streamErrorEvent(error)), ...(announced === undefined ? [] : [eventFrame(streamFailedEvent(announced, error))])], + use.gateway.dump, + ); return { ...rest, - 'response.chat.clientFrames': move(frames), + 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, - 'response.chat.openaiResponses.rendered': move(framing === 'sse' ? renderSSE(frames) : frames), + 'response.chat.openaiResponses.rendered': move(completed.rendered), 'response.http.status': 200, - [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], clientStream.failed)), + [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], completed.failed)), }; }, }); @@ -168,34 +189,20 @@ const streamFailedEvent = (announced: ClientResponseResource, error: unknown): C } as ClientOpenAIResponsesStreamEvent; }; -/** The client's terminal frames are protocol content, so they precede recording and transport - * framing. A stream that finishes carries its own terminator, while an interrupted one carries - * the failure under the response id already announced to that client. - * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx?plain=1#L84 - */ -const completeStream = ( +const completeEventStream = ( frames: AsyncIterable>, - framing: OpenAIResponsesStreamFraming, reportFailure: (error: unknown) => void, ): { frames: AsyncIterable>; failed: Promise } => { let settle!: (failed: boolean) => void; const failed = new Promise(resolve => { settle = resolve; }); const stream = (async function* () { - let announced: ClientResponseResource | undefined; let failed = false; try { - for await (const frame of frames) { - if (frame.type === 'done') continue; - if ('response' in frame.event) announced = frame.event.response; - yield frame; - } - if (framing === 'sse') yield doneFrame(); + for await (const frame of frames) if (frame.type !== 'done') yield frame; } catch (error) { failed = true; reportFailure(error); - if (framing === 'events') throw error; - yield eventFrame(streamErrorEvent(error)); - if (announced !== undefined) yield eventFrame(streamFailedEvent(announced, error)); + throw error; } finally { settle(failed); } @@ -203,16 +210,9 @@ const completeStream = ( return { frames: { [Symbol.asyncIterator]: () => stream }, failed }; }; -// Persistence and client projection can fail after the upstream terminal has settled its -// usage. Settlement waits for both boundaries and retains the measured upstream quantities. -const withClientVerdict = (reading: Deferred | null, failed: Promise): Deferred | null => - reading === null ? null : defer(Promise.all([reading, failed]).then(([outcome, failed]) => ({ ...outcome, failed: outcome.failed || failed }))); - -const renderSSE = (frames: AsyncIterable>): AsyncIterable => ({ - // The frames the client reads are a reframing of the ones the record holds, so this key - // points at that same stream rather than at nothing. - ...streamReferenceOf(frames), +const completedSseFrames = (frames: AsyncIterable>): AsyncIterable> => ({ [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) yield openaiResponsesProtocolFrameToSSEFrame(frame); + for await (const frame of frames) if (frame.type !== 'done') yield frame; + yield doneFrame(); })(), }); diff --git a/packages/gateway/src/data-plane/chat/shared/client-stream.ts b/packages/gateway/src/data-plane/chat/shared/client-stream.ts new file mode 100644 index 0000000000..11cac696a0 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/client-stream.ts @@ -0,0 +1,49 @@ +import { recordStream, streamReferenceOf, type RunDump } from '../../../dump/run-sink.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import { defer, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; + +export const framedClientStream = >( + source: AsyncIterable, + render: (frame: Frame) => SseFrame, + errorFrames: (error: unknown) => readonly Frame[], + dump: RunDump | null, +): { + readonly frames: AsyncIterable; + readonly rendered: AsyncIterable; + readonly failed: Promise; + readonly release: () => Promise; +} => { + let settle!: (failed: boolean) => void; + const failed = new Promise(resolve => { settle = resolve; }); + let started = false; + const generator = (async function* () { + started = true; + let failed = false; + try { + for await (const frame of source) yield { frame, wire: render(frame) }; + } catch (error) { + failed = true; + dump?.failed(error); + for (const frame of errorFrames(error)) yield { frame, wire: render(frame) }; + } finally { + settle(failed); + } + })(); + // A protocol frame is published only after it has a valid transport representation. + // The pair keeps serialization errors inside completion without formatting a second time. + const packets = recordStream({ [Symbol.asyncIterator]: () => generator }, dump, packet => packet.frame); + const reference = streamReferenceOf(packets); + return { + frames: { ...reference, [Symbol.asyncIterator]: () => (async function* () { for await (const packet of packets) yield packet.frame; })() }, + rendered: { ...reference, [Symbol.asyncIterator]: () => (async function* () { for await (const packet of packets) yield packet.wire; })() }, + failed, + release: async () => { + if (!started) settle(false); + await generator.return(); + }, + }; +}; + +export const withClientVerdict = (reading: Deferred | null, failed: Promise): Deferred | null => + reading === null ? null : defer(Promise.all([reading, failed]).then(([outcome, clientFailed]) => ({ ...outcome, failed: outcome.failed || clientFailed }))); diff --git a/packages/protocols/src/openai-chat-completions/index.ts b/packages/protocols/src/openai-chat-completions/index.ts index ed7ede10ab..28cd408e6a 100644 --- a/packages/protocols/src/openai-chat-completions/index.ts +++ b/packages/protocols/src/openai-chat-completions/index.ts @@ -1,5 +1,11 @@ // OpenAI Chat Completions type definitions (subset needed for translation) +export interface OpenAIChatCompletionsErrorEvent { + error: { message: string; type?: string; name?: string; stack?: string; cause?: unknown; target_api?: string }; +} + +export type ClientOpenAIChatCompletionsStreamEvent = OpenAIChatCompletionsStreamEvent | OpenAIChatCompletionsErrorEvent; + export interface OpenAIChatCompletionsPayload { model: string; messages: OpenAIChatCompletionsMessage[]; diff --git a/packages/protocols/src/openai-chat-completions/to-sse.ts b/packages/protocols/src/openai-chat-completions/to-sse.ts index 47dbbd3497..b00d9d2a3c 100644 --- a/packages/protocols/src/openai-chat-completions/to-sse.ts +++ b/packages/protocols/src/openai-chat-completions/to-sse.ts @@ -1,13 +1,13 @@ -import type { OpenAIChatCompletionsStreamEvent } from './index.ts'; +import type { ClientOpenAIChatCompletionsStreamEvent } from './index.ts'; import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame, sseFrame } from '../common/index.ts'; interface OpenAIChatCompletionsSseFrameOptions { includeUsageChunk: boolean; } -export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: { includeUsageChunk: true }): SseFrame; -export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null; -export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null { +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: { includeUsageChunk: true }): SseFrame; +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null; +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null { if (frame.type === 'done') return sseFrame('[DONE]'); if (!options.includeUsageChunk && isOpenAIUsageOnlyEventShape(frame.event)) return null; return sseFrame(JSON.stringify(frame.event)); From 790d0b8b0d9953770f56bf55f7547aae006349fc Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 05:32:12 +0800 Subject: [PATCH 119/156] refactor(codex): serve Responses through provider stages Prepare parsed private wire content, retain complete Lite mappings, decode and restore target streams in named stages, and observe actual model endpoint calls before response processing. Preserve credential retries, preflight outcomes, compaction projection and provider-owned headers while recording quota work as deferred facts. --- .../provider-codex/__tests__/provider_test.ts | 54 +++++++++++++++++-- packages/provider-codex/src/fetch.ts | 33 +++++++++--- .../inject-default-instructions.ts | 12 +++-- .../strip-unsupported-fields.ts | 7 +-- packages/provider-codex/src/pipeline-facts.ts | 3 +- packages/provider-codex/src/pipelines.ts | 16 ++++-- packages/provider-codex/src/provider.ts | 2 +- .../provider-codex/src/stages/authenticate.ts | 8 +-- .../src/stages/inject-default-instructions.ts | 17 ++++++ .../src/stages/normalize-responses-headers.ts | 28 ++++++++++ .../src/stages/observe-response.ts | 9 +++- .../src/stages/prepare-request.ts | 3 +- .../src/stages/prepare-responses.ts | 45 ++++++++++++++++ .../src/stages/restore-responses-lite.ts | 24 +++++++++ .../src/stages/restore-responses-output.ts | 19 +++++++ .../provider-codex/src/stages/retry-access.ts | 2 +- .../strip-unsupported-response-fields.ts | 17 ++++++ 17 files changed, 268 insertions(+), 31 deletions(-) create mode 100644 packages/provider-codex/src/stages/inject-default-instructions.ts create mode 100644 packages/provider-codex/src/stages/normalize-responses-headers.ts create mode 100644 packages/provider-codex/src/stages/prepare-responses.ts create mode 100644 packages/provider-codex/src/stages/restore-responses-lite.ts create mode 100644 packages/provider-codex/src/stages/restore-responses-output.ts create mode 100644 packages/provider-codex/src/stages/strip-unsupported-response-fields.ts diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index be745e91bb..a5e8df44a9 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -7,7 +7,7 @@ import type { CodexAccessTokenEntry, CodexUpstreamState } from '../src/state.ts' import { exchangeResponse } from '@floway-dev/http/pipeline'; import { move, run, setRelease, type Event } from '@floway-dev/pipeline'; import { directFetcher, initProviderRepo, providerModelFacts, type UpstreamRecord } from '@floway-dev/provider'; -import { callProviderPipeline, noopUpstreamCallOptions, readJsonRequest, stubProviderModel } from '@floway-dev/test-utils'; +import { callProviderPipeline, collectChatProviderPipeline, noopUpstreamCallOptions, readJsonRequest, stubProviderModel } from '@floway-dev/test-utils'; const farFutureMs = Date.now() + 24 * 60 * 60 * 1000; @@ -85,14 +85,62 @@ const idToken = (planType = 'plus'): string => [ Buffer.from('signature').toString('base64url'), ].join('.'); -const oauthTokenResponse = (overrides: Partial<{ access_token: string; refresh_token: string; expires_in: number }> = {}): Response => new Response(JSON.stringify({ +const oauthTokenResponse = (overrides: Partial<{ access_token: string; refresh_token: string; expires_in: number; id_token: string }> = {}): Response => new Response(JSON.stringify({ access_token: overrides.access_token ?? 'at_minted', refresh_token: overrides.refresh_token ?? 'rt_v2', - id_token: idToken(), + id_token: overrides.id_token ?? idToken(), expires_in: overrides.expires_in ?? 3600, }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); describe('createCodexProvider', () => { + test.each(['openaiResponses', 'openaiResponsesCompact'] as const)('%s pipeline keeps parsed private wire content and actual endpoint observations', async operation => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => operation === 'openaiResponses' + ? new Response(sseResponse().body, { headers: { [CODEX_RESPONSES_LITE_HEADER]: 'true' } }) + : Response.json({ id: 'compact', object: 'response.compaction', output: [] })); + const provider = createCodexProvider(current!); + const model = stubProviderModel({ id: 'future-lite', endpoints: { openaiResponses: {} }, providerData: { useResponsesLite: true } }); + const options = noopUpstreamCallOptions(); + const result = await collectChatProviderPipeline(provider, operation, model, { + input: [{ type: 'message', role: 'user', content: 'hi' }], tools: [{ type: 'function', name: 'lookup', parameters: { type: 'object' } }], text: { verbosity: 'low' }, + }, undefined, options); + const wire = await readJsonRequest(fetchSpy.mock.calls[0]![1] as RequestInit) as Record; + expect(wire).not.toHaveProperty('instructions'); + expect(wire).not.toHaveProperty('tools'); + expect(wire.text).toEqual({ verbosity: 'low' }); + expect(typeof result.facts['request.http.body']).toBe('object'); + expect(result.facts['response.provider.called']).toBe(true); + expect(result.facts['response.provider.modelKey']).toBe('future-lite'); + const output = result.facts['response.provider.output']; + if (output === null || !('kind' in output)) throw new Error('Expected protocol output'); + if (output.kind === 'stream') { + expect(result.frames.filter(frame => frame.type === 'event').map(frame => frame.event.type)).toEqual(['response.created', 'response.in_progress', 'response.completed']); + const exchange = result.facts['response.http.exchange']; + if (exchange.type !== 'response') throw new Error('Expected response'); + expect(exchange.headers.some(([name]) => name === CODEX_RESPONSES_LITE_HEADER)).toBe(false); + expect(exchange.headers).toContainEqual(['content-type', 'text/event-stream']); + } else expect(output.body).toMatchObject({ object: 'response.compaction' }); + }); + + test('Responses pipeline retains the first 401 observation and retries without applying the image plan gate', async () => { + const state = current!.state as CodexUpstreamState; + current = { ...current!, config: { accounts: [{ email: 'a@b.com', chatgptAccountId: 'acc', chatgptUserId: 'usr', planType: 'free' }] }, state: { accounts: [{ ...state.accounts[0]!, accessToken: { ...freshAccessToken, planType: 'free' } }] } }; + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + if (String(input) === 'https://auth.openai.com/oauth/token') return oauthTokenResponse({ id_token: idToken('free') }); + return new Headers(init?.headers).get('authorization') === 'Bearer at' + ? Response.json({ error: { code: 'invalid_token', message: 'expired' } }, { status: 401 }) + : sseResponse(); + }); + const provider = createCodexProvider(current!); + const model = stubProviderModel({ id: 'gpt-5.4', endpoints: { openaiResponses: {} } }); + const options = noopUpstreamCallOptions(); + const result = await collectChatProviderPipeline(provider, 'openaiResponses', model, { input: [] }, undefined, options); + expect(result.facts['response.provider.previousCalls']).toEqual([{ modelKey: 'gpt-5.4' }]); + expect(fetchSpy.mock.calls.filter(([url]) => String(url).includes('/responses'))).toHaveLength(2); + const output = result.facts['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'stream') throw new Error('Expected stream'); + expect(result.frames.some(frame => frame.type === 'event' && frame.event.type === 'response.completed')).toBe(true); + }); + test('Codex Images records the complete parsed 401 body and preserves its preceding endpoint observation', async () => { const refusal = { error: { code: 'invalid_token', message: 'expired bearer' }, diagnostic: 'x'.repeat(600) }; vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { diff --git a/packages/provider-codex/src/fetch.ts b/packages/provider-codex/src/fetch.ts index cd87a49328..dc4a0cbc02 100644 --- a/packages/provider-codex/src/fetch.ts +++ b/packages/provider-codex/src/fetch.ts @@ -399,10 +399,10 @@ interface PreparedCodexResponsesRequest { body: ReplayableBody; } -const prepareCodexResponsesRequest = ( +export const prepareCodexResponsesContent = ( opts: CallCodexOpenAIResponsesOptions | CallCodexOpenAIResponsesCompactOptions, action: 'generate' | 'compact', -): PreparedCodexResponsesRequest => { +): Omit & { body: Record } => { const clientMetadata = { ...clientCodexClientMetadata(opts.body) }; delete clientMetadata[CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY]; const clientTurnMetadata = callerTurnMetadata(opts, clientMetadata); @@ -422,12 +422,20 @@ const prepareCodexResponsesRequest = ( identity, turnMetadataJson, lite, - body: jsonRequestBody(action === 'compact' + body: action === 'compact' ? buildCodexOpenAIResponsesCompactBody(wire, opts.model.id) - : buildCodexOpenAIResponsesBody(wire, opts.model.id, identity, turnMetadataJson.body)), + : buildCodexOpenAIResponsesBody(wire, opts.model.id, identity, turnMetadataJson.body), }; }; +const prepareCodexResponsesRequest = ( + opts: CallCodexOpenAIResponsesOptions | CallCodexOpenAIResponsesCompactOptions, + action: 'generate' | 'compact', +): PreparedCodexResponsesRequest => { + const prepared = prepareCodexResponsesContent(opts, action); + return { ...prepared, body: jsonRequestBody(prepared.body) }; +}; + const buildCodexOpenAIResponsesBody = ( wire: CodexResponsesBody, modelKey: string, @@ -570,15 +578,26 @@ export const classifyCodexHttpResponse = async ( return response; }; +export const writeCodexQuotaObservation = ( + opts: CodexBackendCallBase, + response: Response, + isRateLimited: boolean, + policy: 'always' | 'when-present', +): Promise | null => { + const snapshot = parseCodexQuotaHeaders(response.headers, { now: new Date(), isRateLimited }); + return policy === 'when-present' && !hasCodexQuotaReading(snapshot) + ? null + : putCodexQuota(opts.upstreamId, opts.account.chatgptAccountId, snapshot); +}; + const persistCodexQuotaObservation = ( opts: CodexBackendCallBase, response: Response, isRateLimited: boolean, policy: 'always' | 'when-present', ): void => { - const snapshot = parseCodexQuotaHeaders(response.headers, { now: new Date(), isRateLimited }); - if (policy === 'when-present' && !hasCodexQuotaReading(snapshot)) return; - registerBackgroundWrite(opts, putCodexQuota(opts.upstreamId, opts.account.chatgptAccountId, snapshot)); + const write = writeCodexQuotaObservation(opts, response, isRateLimited, policy); + if (write !== null) registerBackgroundWrite(opts, write); }; const dispatchCodexImageCall = async ( diff --git a/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts b/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts index 999ef066e7..0b9a6d189b 100644 --- a/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts +++ b/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts @@ -4,13 +4,17 @@ import type { OpenAIResponsesBoundaryCtx } from './types.ts'; // Native and translated callers may omit the field, so the provider supplies a // neutral value at its boundary. Other values remain upstream-owned validation. // https://github.com/im4codes/imcodes/blob/5f769d933dfd679e3a4d670183b0384a1baf62cd/src/agent/providers/codex-sdk.ts#L560-L579 +export const defaultInstructionsPayload =

>(payload: P): P => { + const instructions = payload.instructions; + return instructions === undefined || instructions === null || instructions === '' + ? { ...payload, instructions: "You're a helpful assistant." } + : payload; +}; + export const injectDefaultInstructions = async ( ctx: OpenAIResponsesBoundaryCtx, run: () => Promise, ): Promise => { - const instructions = ctx.payload.instructions; - if (instructions === undefined || instructions === null || instructions === '') { - ctx.payload = { ...ctx.payload, instructions: "You're a helpful assistant." }; - } + ctx.payload = defaultInstructionsPayload(ctx.payload); return await run(); }; diff --git a/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts b/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts index 497930df35..e4b2ba9c9e 100644 --- a/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts +++ b/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts @@ -20,12 +20,13 @@ const CODEX_UNSUPPORTED_BODY_FIELDS = [ 'stream_options', ] as const; +export const supportedResponsesPayload =

>(payload: P): P => + Object.fromEntries(Object.entries(payload).filter(([key]) => !(CODEX_UNSUPPORTED_BODY_FIELDS as readonly string[]).includes(key))) as P; + export const stripUnsupportedFields = async ( ctx: OpenAIResponsesBoundaryCtx, run: () => Promise, ): Promise => { - const next: Record = { ...(ctx.payload as unknown as Record) }; - for (const key of CODEX_UNSUPPORTED_BODY_FIELDS) delete next[key]; - ctx.payload = next as unknown as typeof ctx.payload; + ctx.payload = supportedResponsesPayload(ctx.payload); return await run(); }; diff --git a/packages/provider-codex/src/pipeline-facts.ts b/packages/provider-codex/src/pipeline-facts.ts index 7edaa35088..bb443e1fe4 100644 --- a/packages/provider-codex/src/pipeline-facts.ts +++ b/packages/provider-codex/src/pipeline-facts.ts @@ -5,7 +5,7 @@ import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; import { secret, type Secret } from '@floway-dev/pipeline'; import type { ProviderModelFacts, ProviderOperationPayloads, ProviderRequest, ProviderServices } from '@floway-dev/provider'; -export type CodexOperation = 'alphaSearch' | 'openaiImagesGenerations' | 'openaiImagesEdits'; +export type CodexOperation = 'alphaSearch' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiResponses' | 'openaiResponsesCompact'; export type CodexRequest = ProviderRequest; export type CodexToken = Omit & { readonly token: Secret }; export type CodexAccount = Omit & { @@ -25,6 +25,7 @@ export interface CodexHttpFacts extends HttpRequestFacts { 'request.codex.accessToken': CodexToken; 'request.codex.plan': CodexPlanObservation | null; 'request.codex.modelKey': string; + 'request.provider.modelKey': string; } export interface CodexPipelineConfig { diff --git a/packages/provider-codex/src/pipelines.ts b/packages/provider-codex/src/pipelines.ts index 330a78d19f..53274655e8 100644 --- a/packages/provider-codex/src/pipelines.ts +++ b/packages/provider-codex/src/pipelines.ts @@ -1,16 +1,24 @@ import type { CodexPipelineConfig, CodexRequest } from './pipeline-facts.ts'; import { authenticateCodex } from './stages/authenticate.ts'; import { authorizeCodexHeaders } from './stages/authorize-headers.ts'; +import { injectCodexDefaultInstructions } from './stages/inject-default-instructions.ts'; +import { normalizeCodexResponsesHeaders } from './stages/normalize-responses-headers.ts'; import { observeCodexResponse } from './stages/observe-response.ts'; import { prepareCodexRequest } from './stages/prepare-request.ts'; +import { prepareCodexResponses } from './stages/prepare-responses.ts'; import { readCodexAccount } from './stages/read-account.ts'; +import { restoreCodexResponsesLite } from './stages/restore-responses-lite.ts'; +import { restoreCodexResponsesOutput } from './stages/restore-responses-output.ts'; import { retryCodexAccess } from './stages/retry-access.ts'; +import { stripCodexUnsupportedFields } from './stages/strip-unsupported-response-fields.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderPipelines, ProviderResponse } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, selectProviderResponsesAction, type ProviderChatResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; export const createCodexPipelines = (config: CodexPipelineConfig): ProviderPipelines => ({ - alphaSearch: compose, ProviderResponse>('codex.alphaSearch', [readCodexAccount<'alphaSearch'>(config), authenticateCodex(config, 'alphaSearch'), prepareCodexRequest('alphaSearch'), retryCodexAccess(config, 'alphaSearch'), authorizeCodexHeaders, observeCodexResponse(config, 'alphaSearch'), http]), - openaiImagesGenerations: compose, ProviderResponse>('codex.openaiImagesGenerations', [readCodexAccount<'openaiImagesGenerations'>(config), authenticateCodex(config, 'openaiImagesGenerations'), prepareCodexRequest('openaiImagesGenerations'), retryCodexAccess(config, 'openaiImagesGenerations'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesGenerations'), http]), - openaiImagesEdits: compose, ProviderResponse>('codex.openaiImagesEdits', [readCodexAccount<'openaiImagesEdits'>(config), authenticateCodex(config, 'openaiImagesEdits'), prepareCodexRequest('openaiImagesEdits'), retryCodexAccess(config, 'openaiImagesEdits'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesEdits'), http]), + openaiResponses: compose, ProviderChatResponse<'openaiResponses'>>('codex.openaiResponses', [selectProviderResponsesAction('generate'), readCodexAccount<'openaiResponses'>(config), authenticateCodex(config, 'openaiResponses'), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponses(config, 'openaiResponses'), restoreCodexResponsesOutput, restoreCodexResponsesLite, decodeProviderResponse('openaiResponses'), retryCodexAccess(config, 'openaiResponses'), authorizeCodexHeaders, normalizeCodexResponsesHeaders, observeCodexResponse(config, 'openaiResponses'), observeProviderCall, http]), + openaiResponsesCompact: compose, ProviderChatResponse<'openaiResponsesCompact'>>('codex.openaiResponsesCompact', [selectProviderResponsesAction('compact'), readCodexAccount<'openaiResponsesCompact'>(config), authenticateCodex(config, 'openaiResponsesCompact'), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponses(config, 'openaiResponsesCompact'), restoreCodexResponsesLite, decodeProviderResponse('openaiResponsesCompact'), retryCodexAccess(config, 'openaiResponsesCompact'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiResponsesCompact'), observeProviderCall, http]), + alphaSearch: compose, ProviderResponse>('codex.alphaSearch', [readCodexAccount<'alphaSearch'>(config), authenticateCodex(config, 'alphaSearch'), prepareCodexRequest('alphaSearch'), retryCodexAccess(config, 'alphaSearch'), authorizeCodexHeaders, observeCodexResponse(config, 'alphaSearch'), observeProviderCall, http]), + openaiImagesGenerations: compose, ProviderResponse>('codex.openaiImagesGenerations', [readCodexAccount<'openaiImagesGenerations'>(config), authenticateCodex(config, 'openaiImagesGenerations'), prepareCodexRequest('openaiImagesGenerations'), retryCodexAccess(config, 'openaiImagesGenerations'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesGenerations'), observeProviderCall, http]), + openaiImagesEdits: compose, ProviderResponse>('codex.openaiImagesEdits', [readCodexAccount<'openaiImagesEdits'>(config), authenticateCodex(config, 'openaiImagesEdits'), prepareCodexRequest('openaiImagesEdits'), retryCodexAccess(config, 'openaiImagesEdits'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesEdits'), observeProviderCall, http]), }); diff --git a/packages/provider-codex/src/provider.ts b/packages/provider-codex/src/provider.ts index f421bfba11..d4367f60c0 100644 --- a/packages/provider-codex/src/provider.ts +++ b/packages/provider-codex/src/provider.ts @@ -4,9 +4,9 @@ import { assertCodexUpstreamRecord, type CodexUpstreamConfig } from './config.ts import { CODEX_DEFAULT_FLAGS } from './defaults.ts'; import { callCodexAlphaSearch, callCodexOpenAIImagesEdits, callCodexOpenAIImagesGenerations, callCodexOpenAIResponses, callCodexOpenAIResponsesCompact, type CodexCallEffects } from './fetch.ts'; import { CODEX_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; -import { createCodexPipelines } from './pipelines.ts'; import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { codexImageProviderModel, codexPlanSupportsImages, codexRawToProviderModel, fetchCodexCatalog } from './models.ts'; +import { createCodexPipelines } from './pipelines.ts'; import { assertCodexUpstreamState, findCodexAccountIndex, persistCodexRefreshTokenRotation, persistCodexTerminalState } from './state.ts'; import { runInterceptors } from '@floway-dev/interceptor'; import { getProviderRepo, resolveEffectiveFlags, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type UpstreamRecord } from '@floway-dev/provider'; diff --git a/packages/provider-codex/src/stages/authenticate.ts b/packages/provider-codex/src/stages/authenticate.ts index 0adb1dbe20..d7f192d627 100644 --- a/packages/provider-codex/src/stages/authenticate.ts +++ b/packages/provider-codex/src/stages/authenticate.ts @@ -6,24 +6,24 @@ import { takeHttpResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, type Handed } from '@floway-dev/pipeline'; import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; -export const authenticateCodex = (config: CodexPipelineConfig, operation: O) => defineStage, CodexAuthenticatedFacts, ProviderResponse, ProviderResponse, ProviderResponse, ProviderServices>({ +export const authenticateCodex = (config: CodexPipelineConfig, operation: O) => defineStage, CodexAuthenticatedFacts, ProviderResponse, ProviderResponse, ProviderResponse & { 'response.provider.output': null }, ProviderServices>({ name: 'authenticateCodex', through: { request: { needs: ['request.codex.account', 'request.provider.model', 'request.http.callId'], consumes: [], provides: ['request.codex.accessToken', 'request.codex.plan'] }, response: { needs: [], consumes: [], provides: [] }, }, - return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.output'] }, execute: async (facts, next, use) => { const model = facts['request.provider.model']; const ready = await prepareCodexCall(codexCall(config, facts, use, model)); const failed = (response: Response) => { const exchange = takeHttpResponse(response); - return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': model.id, 'response.provider.called': false, 'response.provider.previousCalls': [] }); + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': model.id, 'response.provider.called': false, 'response.provider.previousCalls': [], 'response.provider.output': null }); }; if (!ready.ok) return failed(ready.response); const observedPlan = codexPlanObservation(ready.accessToken); const plan = observedPlan ?? (config.fallbackPlanType === undefined ? null : { planType: config.fallbackPlanType }); - if (operation !== 'alphaSearch' && plan !== null && !codexPlanSupportsImages(plan.planType)) { + if ((operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') && plan !== null && !codexPlanSupportsImages(plan.planType)) { return failed(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); } return await next(move({ ...facts, 'request.codex.accessToken': tokenFacts(ready.accessToken), 'request.codex.plan': plan })) as Handed; diff --git a/packages/provider-codex/src/stages/inject-default-instructions.ts b/packages/provider-codex/src/stages/inject-default-instructions.ts new file mode 100644 index 0000000000..3e09c48bb8 --- /dev/null +++ b/packages/provider-codex/src/stages/inject-default-instructions.ts @@ -0,0 +1,17 @@ +import { defaultInstructionsPayload } from '../interceptors/openai-responses/inject-default-instructions.ts'; +import type { CodexAccountFacts } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +type Request = CodexAccountFacts<'openaiResponses' | 'openaiResponsesCompact'>; + +export const injectCodexDefaultInstructions = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'injectCodexDefaultInstructions', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => move({ + ...await next(move({ ...facts, 'request.provider.payload': defaultInstructionsPayload(facts['request.provider.payload']) })), + }), +}); diff --git a/packages/provider-codex/src/stages/normalize-responses-headers.ts b/packages/provider-codex/src/stages/normalize-responses-headers.ts new file mode 100644 index 0000000000..2f249413bf --- /dev/null +++ b/packages/provider-codex/src/stages/normalize-responses-headers.ts @@ -0,0 +1,28 @@ +import { CODEX_RESPONSES_LITE_HEADER } from '../constants.ts'; +import type { CodexResponsesHttpFacts } from './prepare-responses.ts'; +import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { isEventStreamMediaType } from '@floway-dev/protocols/common'; + +// The subscription endpoint can return valid SSE without the media-type header. +// Preserve the body lease and normalize the metadata before protocol decoding. +export const normalizeCodexResponsesHeaders = defineStage({ + name: 'normalizeCodexResponsesHeaders', + through: { + request: { needs: ['request.provider.responsesAction'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange'], consumes: ['response.http.exchange'], provides: ['response.http.exchange'] }, + }, + execute: async (facts, next) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + if (facts['request.provider.responsesAction'] === 'compact' || exchange.type !== 'response' || exchange.status < 200 || exchange.status >= 300) return move({ ...back }); + const contentType = exchange.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1]; + const headers = exchange.headers.filter(([name]) => name.toLowerCase() !== CODEX_RESPONSES_LITE_HEADER); + if (!isEventStreamMediaType(contentType)) { + const normalized = headers.filter(([name]) => name.toLowerCase() !== 'content-type'); + normalized.push(['content-type', 'text/event-stream']); + return move({ ...back, 'response.http.exchange': { ...exchange, headers: normalized } }); + } + return move({ ...back, 'response.http.exchange': { ...exchange, headers } }); + }, +}); diff --git a/packages/provider-codex/src/stages/observe-response.ts b/packages/provider-codex/src/stages/observe-response.ts index 72078ff26a..cd2d005dbd 100644 --- a/packages/provider-codex/src/stages/observe-response.ts +++ b/packages/provider-codex/src/stages/observe-response.ts @@ -1,4 +1,4 @@ -import { classifyCodexHttpResponse, classifyCodexUnauthorizedResponse, decodeCodexUpstreamError } from '../fetch.ts'; +import { classifyCodexUnauthorizedResponse, decodeCodexUpstreamError, writeCodexQuotaObservation } from '../fetch.ts'; import { codexCall, type CodexHttpFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; import { exchangeResponse, takeHttpResponse, type HttpResponseFacts } from '@floway-dev/http/pipeline'; import { defer, defineStage, move, setRelease, type Deferred } from '@floway-dev/pipeline'; @@ -27,7 +27,12 @@ export const observeCodexResponse = (config: CodexPipelineConfig, operation: Cod failureBody = parsed.body; classified = await classifyCodexUnauthorizedResponse(call, response, parsed); } else { - classified = await classifyCodexHttpResponse({ ...call, call: { ...call.call, waitUntil: work => { pending.push(work); } } }, response, operation === 'alphaSearch' ? 'always' : 'when-present'); + classified = response; + if (response.ok || response.status === 429) { + const policy = operation === 'alphaSearch' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? 'always' : 'when-present'; + const work = writeCodexQuotaObservation(call, response, response.status === 429, policy); + if (work !== null) pending.push(work); + } } const retained = classified === response ? exchange : takeHttpResponse(classified); return move({ ...back, 'response.http.exchange': retained, 'response.http.body': retained.body, 'response.codex.background': defer(Promise.all(pending)), 'response.codex.failureBody': failureBody }); diff --git a/packages/provider-codex/src/stages/prepare-request.ts b/packages/provider-codex/src/stages/prepare-request.ts index 993fb4c0b6..9518900c29 100644 --- a/packages/provider-codex/src/stages/prepare-request.ts +++ b/packages/provider-codex/src/stages/prepare-request.ts @@ -16,7 +16,7 @@ export const prepareCodexRequest = (operation: O) => d request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.codex.account', 'request.codex.accessToken', 'request.codex.plan'], consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.codex.modelKey', 'request.codex.model'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.codex.modelKey', 'request.codex.model', 'request.provider.modelKey'], }, response: { needs: ['response.http.exchange', 'response.provider.called', 'response.provider.previousCalls'], consumes: [], provides: ['response.provider.modelKey'] }, }, @@ -50,6 +50,7 @@ export const prepareCodexRequest = (operation: O) => d ...rest, 'request.codex.model': model, 'request.codex.modelKey': model.id, + 'request.provider.modelKey': model.id, 'request.http.url': `${CODEX_BACKEND_BASE}${path}`, 'request.http.method': 'POST', 'request.http.headers': headers, diff --git a/packages/provider-codex/src/stages/prepare-responses.ts b/packages/provider-codex/src/stages/prepare-responses.ts new file mode 100644 index 0000000000..341eeb1acd --- /dev/null +++ b/packages/provider-codex/src/stages/prepare-responses.ts @@ -0,0 +1,45 @@ +import { CODEX_BACKEND_BASE, CODEX_CLI_VERSION, CODEX_OPENAI_RESPONSES_COMPACT_PATH, CODEX_OPENAI_RESPONSES_PATH, CODEX_ORIGINATOR, CODEX_RESPONSES_LITE_HEADER, CODEX_USER_AGENT } from '../constants.ts'; +import { prepareCodexResponsesContent } from '../fetch.ts'; +import { codexCall, type CodexAuthenticatedFacts, type CodexHttpFacts, type CodexPipelineConfig } from '../pipeline-facts.ts'; +import type { CodexResponsesLiteRequest } from '../responses-lite.ts'; +import { defineStage, isSecret, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +export interface CodexResponsesHttpFacts extends CodexHttpFacts { + 'request.provider.responsesAction': 'generate' | 'compact'; + 'request.codex.responsesLite': CodexResponsesLiteRequest | null; +} + +export const prepareCodexResponses = (config: CodexPipelineConfig, _operation: O) => defineStage & Pick, CodexResponsesHttpFacts, ProviderChatResponse, ProviderChatResponse, ProviderChatServices>({ + name: 'prepareCodexResponses', + through: { + request: { + needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.codex.account', 'request.codex.accessToken', 'request.codex.plan', 'request.provider.responsesAction'], + consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.codex.model', 'request.codex.modelKey', 'request.provider.modelKey', 'request.provider.responsesAction', 'request.codex.responsesLite'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => { + const { 'request.provider.model': model, 'request.provider.payload': body, ...rest } = facts; + const action = facts['request.provider.responsesAction']; + const inbound = new Headers(facts['request.http.headers'].map(([name, value]) => [name, isSecret(value) ? value.reveal() : value])); + const prepared = prepareCodexResponsesContent({ ...codexCall(config, facts, use, model), headers: inbound, body }, action); + const { identity } = prepared; + const headers: [string, string][] = [ + ['originator', CODEX_ORIGINATOR], ['user-agent', CODEX_USER_AGENT], ['version', CODEX_CLI_VERSION], + ['accept', action === 'compact' ? 'application/json' : 'text/event-stream'], ['content-type', 'application/json'], + ['session-id', identity.sessionId], ['thread-id', identity.threadId], ['x-client-request-id', identity.clientRequestId], + ['x-codex-window-id', identity.windowId], ['x-codex-turn-metadata', prepared.turnMetadataJson.header], + ]; + if (prepared.lite !== undefined) headers.push([CODEX_RESPONSES_LITE_HEADER, 'true']); + return move({ + ...await next(move({ + ...rest, 'request.codex.model': model, 'request.codex.modelKey': model.id, 'request.provider.modelKey': model.id, + 'request.provider.responsesAction': action, 'request.codex.responsesLite': prepared.lite ?? null, + 'request.http.url': `${CODEX_BACKEND_BASE}${action === 'compact' ? CODEX_OPENAI_RESPONSES_COMPACT_PATH : CODEX_OPENAI_RESPONSES_PATH}`, + 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': prepared.body, 'request.http.encoding': 'json' as const, + })), + }); + }, +}); diff --git a/packages/provider-codex/src/stages/restore-responses-lite.ts b/packages/provider-codex/src/stages/restore-responses-lite.ts new file mode 100644 index 0000000000..df00963c45 --- /dev/null +++ b/packages/provider-codex/src/stages/restore-responses-lite.ts @@ -0,0 +1,24 @@ +import { restoreCodexResponsesCompactionResult, restoreCodexResponsesFrames } from '../responses-lite.ts'; +import type { CodexResponsesHttpFacts } from './prepare-responses.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesCompactionResult } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +export const restoreCodexResponsesLite = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'restoreCodexResponsesLite', + through: { + request: { needs: ['request.codex.responsesLite'], consumes: [], provides: [] }, + response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const lite = facts['request.codex.responsesLite']; + const output = back['response.provider.output']; + if (lite === null || output === null || !('kind' in output)) return move({ ...back }); + return move({ + ...back, 'response.provider.output': output.kind === 'value' + ? { kind: 'value' as const, body: restoreCodexResponsesCompactionResult(output.body as OpenAIResponsesCompactionResult, lite.callableIdentities, lite.generatedPrefix) } + : { kind: 'stream' as const, frames: use.recordProtocolFrames(restoreCodexResponsesFrames(output.frames, lite.callableIdentities, lite.requestEchoes)) }, + }); + }, +}); diff --git a/packages/provider-codex/src/stages/restore-responses-output.ts b/packages/provider-codex/src/stages/restore-responses-output.ts new file mode 100644 index 0000000000..f4391918a2 --- /dev/null +++ b/packages/provider-codex/src/stages/restore-responses-output.ts @@ -0,0 +1,19 @@ +import { restoreCodexResponsesOutput as restoreFrames } from '../responses-output.ts'; +import type { CodexResponsesHttpFacts } from './prepare-responses.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +export const restoreCodexResponsesOutput = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'restoreCodexResponsesOutput', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const output = back['response.provider.output']; + return output !== null && 'kind' in output && output.kind === 'stream' + ? move({ ...back, 'response.provider.output': { kind: 'stream' as const, frames: use.recordProtocolFrames(restoreFrames(output.frames)) } }) + : move({ ...back }); + }, +}); diff --git a/packages/provider-codex/src/stages/retry-access.ts b/packages/provider-codex/src/stages/retry-access.ts index da73e6a0fc..52d4095941 100644 --- a/packages/provider-codex/src/stages/retry-access.ts +++ b/packages/provider-codex/src/stages/retry-access.ts @@ -25,7 +25,7 @@ export const retryCodexAccess = (config: CodexPipelineConfig, operation: CodexOp }; if (!fresh.ok) return declined(fresh.response); const plan = codexPlanObservation(fresh.accessToken) ?? facts['request.codex.plan']; - if (operation !== 'alphaSearch' && plan !== null && !codexPlanSupportsImages(plan.planType)) return declined(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); + if ((operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') && plan !== null && !codexPlanSupportsImages(plan.planType)) return declined(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); const retried = await next(move({ ...facts, 'request.codex.accessToken': tokenFacts(fresh.accessToken), 'request.codex.plan': plan })); return move({ ...retried, 'response.provider.called': retried['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }] }); }, diff --git a/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts b/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts new file mode 100644 index 0000000000..75fb8e4b1e --- /dev/null +++ b/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts @@ -0,0 +1,17 @@ +import { supportedResponsesPayload } from '../interceptors/openai-responses/strip-unsupported-fields.ts'; +import type { CodexAccountFacts } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +type Request = CodexAccountFacts<'openaiResponses' | 'openaiResponsesCompact'>; + +export const stripCodexUnsupportedFields = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'stripCodexUnsupportedFields', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => move({ + ...await next(move({ ...facts, 'request.provider.payload': supportedResponsesPayload(facts['request.provider.payload']) })), + }), +}); From 222d5f9d84d2a9bd392437fffd9cf457b97c18f5 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 05:57:57 +0800 Subject: [PATCH 120/156] fix(codex): retain the actual Responses action on unwind Publish the dispatched action before HTTP response processing and retain it on synthetic credential retry outcomes so decoding follows late pipeline action changes. --- packages/provider-codex/src/pipelines.ts | 6 +++--- .../src/stages/normalize-responses-headers.ts | 6 +++--- packages/provider-codex/src/stages/retry-access.ts | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/provider-codex/src/pipelines.ts b/packages/provider-codex/src/pipelines.ts index 53274655e8..5ebd62a93f 100644 --- a/packages/provider-codex/src/pipelines.ts +++ b/packages/provider-codex/src/pipelines.ts @@ -13,11 +13,11 @@ import { retryCodexAccess } from './stages/retry-access.ts'; import { stripCodexUnsupportedFields } from './stages/strip-unsupported-response-fields.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import { decodeProviderResponse, observeProviderCall, selectProviderResponsesAction, type ProviderChatResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction, type ProviderChatResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; export const createCodexPipelines = (config: CodexPipelineConfig): ProviderPipelines => ({ - openaiResponses: compose, ProviderChatResponse<'openaiResponses'>>('codex.openaiResponses', [selectProviderResponsesAction('generate'), readCodexAccount<'openaiResponses'>(config), authenticateCodex(config, 'openaiResponses'), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponses(config, 'openaiResponses'), restoreCodexResponsesOutput, restoreCodexResponsesLite, decodeProviderResponse('openaiResponses'), retryCodexAccess(config, 'openaiResponses'), authorizeCodexHeaders, normalizeCodexResponsesHeaders, observeCodexResponse(config, 'openaiResponses'), observeProviderCall, http]), - openaiResponsesCompact: compose, ProviderChatResponse<'openaiResponsesCompact'>>('codex.openaiResponsesCompact', [selectProviderResponsesAction('compact'), readCodexAccount<'openaiResponsesCompact'>(config), authenticateCodex(config, 'openaiResponsesCompact'), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponses(config, 'openaiResponsesCompact'), restoreCodexResponsesLite, decodeProviderResponse('openaiResponsesCompact'), retryCodexAccess(config, 'openaiResponsesCompact'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiResponsesCompact'), observeProviderCall, http]), + openaiResponses: compose, ProviderChatResponse<'openaiResponses'>>('codex.openaiResponses', [selectProviderResponsesAction('generate'), readCodexAccount<'openaiResponses'>(config), authenticateCodex(config, 'openaiResponses'), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponses(config, 'openaiResponses'), restoreCodexResponsesOutput, restoreCodexResponsesLite, decodeProviderResponse('openaiResponses'), retryCodexAccess(config, 'openaiResponses'), authorizeCodexHeaders, normalizeCodexResponsesHeaders, observeCodexResponse(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderChatResponse<'openaiResponsesCompact'>>('codex.openaiResponsesCompact', [selectProviderResponsesAction('compact'), readCodexAccount<'openaiResponsesCompact'>(config), authenticateCodex(config, 'openaiResponsesCompact'), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponses(config, 'openaiResponsesCompact'), restoreCodexResponsesLite, decodeProviderResponse('openaiResponsesCompact'), retryCodexAccess(config, 'openaiResponsesCompact'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), alphaSearch: compose, ProviderResponse>('codex.alphaSearch', [readCodexAccount<'alphaSearch'>(config), authenticateCodex(config, 'alphaSearch'), prepareCodexRequest('alphaSearch'), retryCodexAccess(config, 'alphaSearch'), authorizeCodexHeaders, observeCodexResponse(config, 'alphaSearch'), observeProviderCall, http]), openaiImagesGenerations: compose, ProviderResponse>('codex.openaiImagesGenerations', [readCodexAccount<'openaiImagesGenerations'>(config), authenticateCodex(config, 'openaiImagesGenerations'), prepareCodexRequest('openaiImagesGenerations'), retryCodexAccess(config, 'openaiImagesGenerations'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesGenerations'), observeProviderCall, http]), openaiImagesEdits: compose, ProviderResponse>('codex.openaiImagesEdits', [readCodexAccount<'openaiImagesEdits'>(config), authenticateCodex(config, 'openaiImagesEdits'), prepareCodexRequest('openaiImagesEdits'), retryCodexAccess(config, 'openaiImagesEdits'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesEdits'), observeProviderCall, http]), diff --git a/packages/provider-codex/src/stages/normalize-responses-headers.ts b/packages/provider-codex/src/stages/normalize-responses-headers.ts index 2f249413bf..f901c40fe4 100644 --- a/packages/provider-codex/src/stages/normalize-responses-headers.ts +++ b/packages/provider-codex/src/stages/normalize-responses-headers.ts @@ -6,16 +6,16 @@ import { isEventStreamMediaType } from '@floway-dev/protocols/common'; // The subscription endpoint can return valid SSE without the media-type header. // Preserve the body lease and normalize the metadata before protocol decoding. -export const normalizeCodexResponsesHeaders = defineStage({ +export const normalizeCodexResponsesHeaders = defineStage({ name: 'normalizeCodexResponsesHeaders', through: { request: { needs: ['request.provider.responsesAction'], consumes: [], provides: [] }, - response: { needs: ['response.http.exchange'], consumes: ['response.http.exchange'], provides: ['response.http.exchange'] }, + response: { needs: ['response.http.exchange', 'response.provider.responsesAction'], consumes: ['response.http.exchange'], provides: ['response.http.exchange'] }, }, execute: async (facts, next) => { const back = await next(move({ ...facts })); const exchange = back['response.http.exchange']; - if (facts['request.provider.responsesAction'] === 'compact' || exchange.type !== 'response' || exchange.status < 200 || exchange.status >= 300) return move({ ...back }); + if (back['response.provider.responsesAction'] === 'compact' || exchange.type !== 'response' || exchange.status < 200 || exchange.status >= 300) return move({ ...back }); const contentType = exchange.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1]; const headers = exchange.headers.filter(([name]) => name.toLowerCase() !== CODEX_RESPONSES_LITE_HEADER); if (!isEventStreamMediaType(contentType)) { diff --git a/packages/provider-codex/src/stages/retry-access.ts b/packages/provider-codex/src/stages/retry-access.ts index 52d4095941..bd8cd0662d 100644 --- a/packages/provider-codex/src/stages/retry-access.ts +++ b/packages/provider-codex/src/stages/retry-access.ts @@ -6,11 +6,11 @@ import { takeHttpResponse, type HttpResponseFacts } from '@floway-dev/http/pipel import { defineStage, move } from '@floway-dev/pipeline'; import type { ProviderCallResponse, ProviderServices } from '@floway-dev/provider'; -export const retryCodexAccess = (config: CodexPipelineConfig, operation: CodexOperation) => defineStage({ +export const retryCodexAccess = (config: CodexPipelineConfig, operation: CodexOperation) => defineStage({ name: 'retryCodexAccess', through: { request: { needs: ['request.codex.account', 'request.codex.accessToken', 'request.codex.plan', 'request.codex.model', 'request.codex.modelKey', 'request.http.callId'], consumes: ['request.codex.accessToken', 'request.codex.plan'], provides: ['request.codex.accessToken', 'request.codex.plan'] }, - response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls'] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls', ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['response.provider.responsesAction' as const] : [])] }, }, execute: async (facts, next, use) => { const first = await next(move({ ...facts })); @@ -21,12 +21,12 @@ export const retryCodexAccess = (config: CodexPipelineConfig, operation: CodexOp const fresh = await refreshAccessTokenForRetry(codexCall(config, facts, use, facts['request.codex.model']), { ...failedToken, token: failedToken.token.reveal() }, facts['request.codex.plan'] ?? undefined); const declined = (response: Response) => { const nextExchange = takeHttpResponse(response); - return move({ ...first, 'response.http.exchange': nextExchange, 'response.http.body': nextExchange.body, 'response.provider.called': false, 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }] }); + return move({ ...first, 'response.http.exchange': nextExchange, 'response.http.body': nextExchange.body, 'response.provider.called': false, 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }], ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? { 'response.provider.responsesAction': facts['request.provider.responsesAction']! } : {}) }); }; if (!fresh.ok) return declined(fresh.response); const plan = codexPlanObservation(fresh.accessToken) ?? facts['request.codex.plan']; if ((operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') && plan !== null && !codexPlanSupportsImages(plan.planType)) return declined(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); const retried = await next(move({ ...facts, 'request.codex.accessToken': tokenFacts(fresh.accessToken), 'request.codex.plan': plan })); - return move({ ...retried, 'response.provider.called': retried['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }] }); + return move({ ...retried, 'response.provider.called': retried['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }], ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? { 'response.provider.responsesAction': facts['request.provider.responsesAction']! } : {}) }); }, }); From 1ad0e0b64e152a3925aca66b1b1591cdf608cdba Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 05:59:45 +0800 Subject: [PATCH 121/156] refactor(claude-code): serve Messages through provider stages Record shaped detection, immutable mimicry, credential gates, dated model identity and concrete HTTP content in the provider fact space. Retain cached-token retries and full terminal refusal content, and await quota/state work through deferred facts. Preserve incomplete-stream diagnostics through the scoped logger and record parsed target frames before further transforms. --- .../__tests__/provider_test.ts | 37 ++++++++++- packages/provider-claude-code/package.json | 2 + packages/provider-claude-code/src/fetch.ts | 62 +++++++++++-------- .../backfill-required-fields.ts | 16 +++-- .../hoist-user-system-to-messages.ts | 18 +++--- .../inject-billing-block.ts | 11 +++- .../inject-default-template.ts | 23 ++++--- .../inject-identity-block.ts | 9 ++- .../synthesize-metadata-user-id.ts | 21 ++++--- .../anthropic-messages/system-blocks.ts | 4 +- .../src/pipeline-facts.ts | 26 ++++++++ .../provider-claude-code/src/pipelines.ts | 23 +++++++ packages/provider-claude-code/src/provider.ts | 3 +- .../src/stages/authenticate.ts | 47 ++++++++++++++ .../src/stages/backfill-required-fields.ts | 10 +++ .../src/stages/decode-response.ts | 41 ++++++++++++ .../stages/hoist-user-system-to-messages.ts | 10 +++ .../src/stages/inject-billing-block.ts | 10 +++ .../src/stages/inject-default-template.ts | 10 +++ .../src/stages/inject-identity-block.ts | 10 +++ .../src/stages/observe-response.ts | 41 ++++++++++++ .../src/stages/prepare-request.ts | 29 +++++++++ .../src/stages/recognize-shape.ts | 18 ++++++ .../src/stages/retry-access.ts | 36 +++++++++++ .../src/stages/synthesize-metadata-user-id.ts | 10 +++ pnpm-lock.yaml | 6 ++ 26 files changed, 468 insertions(+), 65 deletions(-) create mode 100644 packages/provider-claude-code/src/pipeline-facts.ts create mode 100644 packages/provider-claude-code/src/pipelines.ts create mode 100644 packages/provider-claude-code/src/stages/authenticate.ts create mode 100644 packages/provider-claude-code/src/stages/backfill-required-fields.ts create mode 100644 packages/provider-claude-code/src/stages/decode-response.ts create mode 100644 packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts create mode 100644 packages/provider-claude-code/src/stages/inject-billing-block.ts create mode 100644 packages/provider-claude-code/src/stages/inject-default-template.ts create mode 100644 packages/provider-claude-code/src/stages/inject-identity-block.ts create mode 100644 packages/provider-claude-code/src/stages/observe-response.ts create mode 100644 packages/provider-claude-code/src/stages/prepare-request.ts create mode 100644 packages/provider-claude-code/src/stages/recognize-shape.ts create mode 100644 packages/provider-claude-code/src/stages/retry-access.ts create mode 100644 packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts diff --git a/packages/provider-claude-code/__tests__/provider_test.ts b/packages/provider-claude-code/__tests__/provider_test.ts index 35c7007fd3..7afcfb14ff 100644 --- a/packages/provider-claude-code/__tests__/provider_test.ts +++ b/packages/provider-claude-code/__tests__/provider_test.ts @@ -6,7 +6,7 @@ import { createClaudeCodeProvider } from '../src/provider.ts'; import type { ClaudeCodeAccessTokenEntry, ClaudeCodeAccountCredential, ClaudeCodeUpstreamState } from '../src/state.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; import { initProviderRepo, type FlagId, type AnthropicMessagesUpstreamCallOptions, type UpstreamRecord } from '@floway-dev/provider'; -import { noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, readJsonRequest } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, readJsonRequest } from '@floway-dev/test-utils'; const upstreamId = 'up_cc_provider'; @@ -176,6 +176,41 @@ describe('createClaudeCodeProvider — factory surface', () => { }); describe('createClaudeCodeProvider — callAnthropicMessages routes through chain', () => { + test.each([false, true])('the actual Messages pipeline preserves shaped=%s content and dated model identity', async shaped => { + const provider = createClaudeCodeProvider(currentRecord); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const userId = JSON.stringify({ device_id: 'd'.repeat(32), account_uuid: '', session_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' }); + const payload = { + max_tokens: 16, messages: [{ role: 'user' as const, content: 'hello' }], + ...(shaped ? { system: [{ type: 'text' as const, text: "You are Claude Code, Anthropic's official CLI for Claude." }], metadata: { user_id: userId } } : {}), + }; + const result = await collectChatProviderPipeline(provider, 'anthropicMessages', sonnetProviderModel, payload, undefined, shaped ? cliClientCallOpts() : noopAnthropicMessagesUpstreamCallOptions()); + expect(result.facts['response.provider.called']).toBe(true); + expect(result.facts['response.provider.modelKey']).toBe('claude-sonnet-4-5-20250929'); + expect(result.frames.some(frame => frame.type === 'event' && frame.event.type === 'message_start')).toBe(true); + const init = fetchSpy.mock.calls[0]![1] as RequestInit; + const body = await readJsonRequest(init) as WireAnthropicMessagesPayload; + expect(body.model).toBe('claude-sonnet-4-5-20250929'); + expect(body.stream).toBe(true); + expect(body.system).toHaveLength(shaped ? 1 : 3); + if (shaped) expect(body.metadata.user_id).toBe(userId); + else expect(body.system[0]!.text).toMatch(/^x-anthropic-billing-header:/); + expect(new Headers(init.headers).get('authorization')).toBe('Bearer at_cached'); + expect(new Headers(init.headers).get('anthropic-beta')).toEqual(shaped ? 'oauth-2025-04-20' : expect.stringContaining('claude-code-20250219')); + }); + + test('the Messages pipeline preserves a complete terminal refusal while recording state persistence', async () => { + const body = { error: { type: 'permission_error', message: 'OAuth authentication is currently not allowed for this organization' }, diagnostic: 'x'.repeat(800) }; + vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(body, { status: 403, headers: { 'x-upstream': 'retained' } })); + const result = await collectChatProviderPipeline(createClaudeCodeProvider(currentRecord), 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [] }, undefined, noopAnthropicMessagesUpstreamCallOptions()); + expect(result.output).toBeNull(); + expect(result.response?.status).toBe(403); + expect(result.response?.headers.get('x-upstream')).toBe('retained'); + expect(await result.response!.json()).toEqual(body); + expect(result.facts['response.claudeCode.failureBody']).toEqual(body); + expect((currentRecord.state as ClaudeCodeUpstreamState).accounts[0]!.state).toBe('refresh_failed'); + }); + test('unshaped request runs the re-mimicry chain (3-block system, pinned UA, metadata.user_id)', async () => { const instance = createClaudeCodeProvider(currentRecord); const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); diff --git a/packages/provider-claude-code/package.json b/packages/provider-claude-code/package.json index 70d9de00a1..af071f58d5 100644 --- a/packages/provider-claude-code/package.json +++ b/packages/provider-claude-code/package.json @@ -10,6 +10,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/interceptor": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", diff --git a/packages/provider-claude-code/src/fetch.ts b/packages/provider-claude-code/src/fetch.ts index 66814f40b0..7e76ef231c 100644 --- a/packages/provider-claude-code/src/fetch.ts +++ b/packages/provider-claude-code/src/fetch.ts @@ -1,7 +1,7 @@ import { ensureClaudeCodeAccessToken, invalidateClaudeCodeAccessToken, type EnsuredAccessToken } from './access-token.ts'; import { ClaudeCodeOAuthSessionTerminatedError } from './auth/oauth.ts'; import { pickClaudeCodeHeaders } from './headers.ts'; -import { logWarn, logInfo } from './log.ts'; +import { logWarn, logInfo, type LogFields } from './log.ts'; import type { ClaudeCodeProviderData } from './models.ts'; import { parseClaudeCodeQuotaHeaders, type ClaudeCodeQuotaSnapshot } from './quota.ts'; import { @@ -22,9 +22,9 @@ import { type ProviderStreamResult, } from '@floway-dev/provider'; -const ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages?beta=true'; -const STREAM_DIAGNOSTIC_FRAME_LIMIT = 3; -const STREAM_DIAGNOSTIC_FRAME_DATA_CHARS = 256; +export const ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages?beta=true'; +export const STREAM_DIAGNOSTIC_FRAME_LIMIT = 3; +export const STREAM_DIAGNOSTIC_FRAME_DATA_CHARS = 256; export interface CallClaudeCodeAnthropicMessagesOptions { upstreamId: string; @@ -43,13 +43,13 @@ export interface CallClaudeCodeAnthropicMessagesOptions { call: AnthropicMessagesUpstreamCallOptions; } -const synthetic503 = (message: string): Response => +export const synthetic503 = (message: string): Response => new Response( JSON.stringify({ error: { type: 'claude_code_upstream_unavailable', message } }), { status: 503, headers: { 'content-type': 'application/json' } }, ); -const synthetic429 = (message: string, retryAtIso: string | null, now: Date): Response => { +export const synthetic429 = (message: string, retryAtIso: string | null, now: Date): Response => { const retryAfterSeconds = retryAtIso === null ? 60 : Math.max(0, Math.ceil((new Date(retryAtIso).getTime() - now.getTime()) / 1000)); @@ -67,12 +67,12 @@ const oneLineError = (error: unknown): string => { return message.length > 512 ? `${message.slice(0, 509)}...` : message; }; -interface StreamDiagnosticFrame { +export interface StreamDiagnosticFrame { event: string | null; data: string; } -const observedClaudeCodeMessagesStream = async function* ( +export const observedClaudeCodeMessagesStream = async function* ( events: AsyncIterable>, options: { upstreamId: string; @@ -81,6 +81,7 @@ const observedClaudeCodeMessagesStream = async function* ( signal: AbortSignal | undefined; frames: StreamDiagnosticFrame[]; rawFrameCount: () => number; + warn?: (message: string, fields: Readonly>) => Promise; }, ): AsyncGenerator> { let terminalEvent: 'message_stop' | 'error' | null = null; @@ -97,18 +98,26 @@ const observedClaudeCodeMessagesStream = async function* ( streamError = error; throw error; } finally { - if (options.signal?.aborted || (terminalEvent !== null && streamError === undefined)) return; - logWarn('claude_code_messages_stream_incomplete', { - upstream_id: options.upstreamId, - model: options.model, - request_id: options.headers.get('request-id'), - cf_ray: options.headers.get('cf-ray'), - trace_response: options.headers.get('traceresponse'), - raw_sse_frames: options.rawFrameCount(), - terminal_event: terminalEvent, - error: streamError === undefined ? null : oneLineError(streamError), - last_sse_frames: JSON.stringify(options.frames), - }); + if (!options.signal?.aborted && !(terminalEvent !== null && streamError === undefined)) { + const fields = { + upstream_id: options.upstreamId, + model: options.model, + request_id: options.headers.get('request-id'), + cf_ray: options.headers.get('cf-ray'), + trace_response: options.headers.get('traceresponse'), + raw_sse_frames: options.rawFrameCount(), + terminal_event: terminalEvent, + error: streamError === undefined ? null : options.warn === undefined ? oneLineError(streamError) : streamError, + last_sse_frames: JSON.stringify(options.frames), + }; + if (options.warn === undefined) logWarn('claude_code_messages_stream_incomplete', { ...fields, error: streamError === undefined ? null : oneLineError(streamError) }); + else { + try { await options.warn('claude_code_messages_stream_incomplete', fields); } catch (recordingError) { + if (streamError === undefined || recordingError === streamError) throw recordingError; + throw new AggregateError([streamError, recordingError], 'Claude Code stream failed and recording also failed', { cause: streamError }); + } + } + } } }; @@ -133,7 +142,7 @@ const observedClaudeCodeMessagesStream = async function* ( // body sentinel) and passes it through verbatim — without a reset we'd // otherwise lock the account out indefinitely because the next request // never fires to refresh the snapshot. -const isRateLimitedNow = ( +export const isRateLimitedNow = ( snapshot: ClaudeCodeQuotaSnapshot | null, now: Date, ): snapshot is ClaudeCodeQuotaSnapshot => { @@ -143,7 +152,7 @@ const isRateLimitedNow = ( return new Date(snapshot.reset).getTime() > now.getTime(); }; -const persistQuotaSnapshot = async (upstreamId: string, snapshot: ClaudeCodeQuotaSnapshot): Promise => { +export const persistQuotaSnapshot = async (upstreamId: string, snapshot: ClaudeCodeQuotaSnapshot, info: (message: string, fields: LogFields) => void | Promise = logInfo): Promise => { // Stamped before the write: the mutator is replayed on a lost race and must // return the same document each time, and this records when the snapshot was // observed rather than which attempt landed it. @@ -166,7 +175,7 @@ const persistQuotaSnapshot = async (upstreamId: string, snapshot: ClaudeCodeQuot // verbatim. Operators care about the moment the upstream flipped from // `allowed` to `rejected` (or back), not the steady state. if (priorStatus !== snapshot.status) { - logInfo('claude_code_quota_state_transition', { + await info('claude_code_quota_state_transition', { upstream_id: upstreamId, account_uuid: previousAccount.accountUuid, from_status: priorStatus, @@ -228,7 +237,7 @@ interface AnthropicErrorBody { // JSON shape doesn't match `{error:{type,message}}`) is the common case // for unrelated errors (`max_tokens` validation, beta-feature gating, // etc.) and must NOT trigger a terminal flip. -const detectTerminalSentinel = (status: number, bodyText: string): string | null => { +export const detectTerminalSentinel = (status: number, bodyText: string): string | null => { if (status !== 400 && status !== 403) return null; let parsed: unknown; try { @@ -263,11 +272,12 @@ const detectTerminalSentinel = (status: number, bodyText: string): string | null // untouched — the repo reads that as "nothing to do" and the dashboard keeps // the first signal. Merging the two helpers would force conditional dispatch // on every one of these axes. -const persistTerminalAccountState = async ( +export const persistTerminalAccountState = async ( upstreamId: string, terminalMessage: string, reason: string, upstreamStatus: number, + warn: (message: string, fields: LogFields) => void | Promise = logWarn, ): Promise => { // Stamped before the write for the same reason as the quota snapshot: a // replay must produce the same document. @@ -286,7 +296,7 @@ const persistTerminalAccountState = async ( })); }); if (previousAccount.state !== 'active') return; - logWarn('claude_code_account_state_flip', { + await warn('claude_code_account_state_flip', { upstream_id: upstreamId, account_uuid: previousAccount.accountUuid, from_state: previousAccount.state, diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts index 1bd7cf3fea..61d2df5943 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts @@ -18,15 +18,19 @@ import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS } from '@floway-dev/protocols/an // // Positioned at the head of the chain so the rest of the re-mimicry steps // see a fully-formed payload. Caller-supplied values are never overwritten. +export const backfillRequiredFieldsPayload =

>(payload: P, model: Pick): P => { + const next = { ...payload }; + + next.max_tokens ??= model.limits.max_output_tokens ?? ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS; + next.temperature ??= 1; + + return next; +}; + export const backfillRequiredFields = async ( ctx: AnthropicMessagesBoundaryCtx, run: () => Promise, ): Promise => { - const next = { ...ctx.payload }; - - next.max_tokens ??= ctx.model.limits.max_output_tokens ?? ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS; - next.temperature ??= 1; - - ctx.payload = next; + ctx.payload = backfillRequiredFieldsPayload(ctx.payload, ctx.model); return await run(); }; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts index c9bc0fdf58..21d5e58c70 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts @@ -27,11 +27,8 @@ const SYNTHETIC_ACK = 'Understood. I will follow these instructions.'; // // References: // - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/gateway_service.go#L4480-L4486 -export const hoistUserSystemToMessages = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - const system: string | AnthropicMessagesTextBlock[] | undefined = ctx.payload.system; +export const hoistUserSystemToMessagesPayload =

>(payload: P): P => { + const system: string | AnthropicMessagesTextBlock[] | undefined = payload.system; let captured = ''; if (typeof system === 'string') { captured = system; @@ -43,7 +40,7 @@ export const hoistUserSystemToMessages = async ( } // inject-billing-block et al rebuild `system` from scratch as a three-block // array; removing the field here keeps the boundary mutation self-contained. - const nextPayload = { ...ctx.payload }; + const nextPayload = { ...payload }; delete nextPayload.system; if (captured !== '') { @@ -58,6 +55,13 @@ export const hoistUserSystemToMessages = async ( nextPayload.messages = [...synthetic, ...nextPayload.messages]; } - ctx.payload = nextPayload; + return nextPayload; +}; + +export const hoistUserSystemToMessages = async ( + ctx: AnthropicMessagesBoundaryCtx, + run: () => Promise, +): Promise => { + ctx.payload = hoistUserSystemToMessagesPayload(ctx.payload); return await run(); }; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts index 3c6cca908b..87f5772e4d 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts @@ -16,12 +16,17 @@ import { CLAUDE_CLI_VERSION } from '../../headers.ts'; // shape Anthropic will actually see on the wire, so the fingerprint must // reflect it — fingerprinting the pre-hoist shape would compute a different // value than what the request body settles to and break CC mimicry. +export const injectBillingBlockPayload =

>(payload: P): P => { + const fingerprint = computeCcVersionFingerprint(CLAUDE_CLI_VERSION, payload); + const block = buildBillingBlock(CLAUDE_CLI_VERSION, fingerprint); + payload = { ...payload, system: [block] }; + return payload; +}; + export const injectBillingBlock = async ( ctx: AnthropicMessagesBoundaryCtx, run: () => Promise, ): Promise => { - const fingerprint = computeCcVersionFingerprint(CLAUDE_CLI_VERSION, ctx.payload); - const block = buildBillingBlock(CLAUDE_CLI_VERSION, fingerprint); - ctx.payload = { ...ctx.payload, system: [block] }; + ctx.payload = injectBillingBlockPayload(ctx.payload); return await run(); }; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts index ee473e524e..927d477998 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts @@ -17,7 +17,7 @@ const ANTHROPIC_CACHE_BREAKPOINT_CAP = 4; // Mirrors sub2api's `collectCacheControlPaths`. The billing and identity // blocks injected earlier in this chain carry no `cache_control`, so they // never contribute here. -const countCacheBreakpoints = (payload: AnthropicMessagesPayload): number => { +const countCacheBreakpoints = (payload: Pick): number => { let count = 0; if (payload.tools) { @@ -57,19 +57,24 @@ const countCacheBreakpoints = (payload: AnthropicMessagesPayload): number => { // // Chain order in ./index.ts guarantees `system` is already a fresh array // when this runs. -export const injectDefaultTemplate = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - if (!Array.isArray(ctx.payload.system)) { +export const injectDefaultTemplatePayload =

>(payload: P): P => { + if (!Array.isArray(payload.system)) { throw new Error('inject-default-template: expected system to be an array (inject-billing-block must run first)'); } - const system = ctx.payload.system; - const callerBreakpoints = countCacheBreakpoints(ctx.payload); + const system = payload.system; + const callerBreakpoints = countCacheBreakpoints(payload); const wouldOverflowBreakpointCap = callerBreakpoints >= ANTHROPIC_CACHE_BREAKPOINT_CAP; const templateBlock = wouldOverflowBreakpointCap ? { type: 'text' as const, text: DEFAULT_TEMPLATE_BLOCK.text } : DEFAULT_TEMPLATE_BLOCK; - ctx.payload = { ...ctx.payload, system: [...system, templateBlock] }; + payload = { ...payload, system: [...system, templateBlock] }; + return payload; +}; + +export const injectDefaultTemplate = async ( + ctx: AnthropicMessagesBoundaryCtx, + run: () => Promise, +): Promise => { + ctx.payload = injectDefaultTemplatePayload(ctx.payload); return await run(); }; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts index fafea88760..1b80776ffa 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts @@ -3,11 +3,16 @@ import type { AnthropicMessagesBoundaryCtx } from './types.ts'; import type { AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; // system[1]; relies on injectBillingBlock having materialized payload.system as an array (see ./index.ts chain order). +export const injectIdentityBlockPayload =

>(payload: P): P => { + const system = payload.system as AnthropicMessagesTextBlock[]; + payload = { ...payload, system: [...system, IDENTITY_BLOCK] }; + return payload; +}; + export const injectIdentityBlock = async ( ctx: AnthropicMessagesBoundaryCtx, run: () => Promise, ): Promise => { - const system = ctx.payload.system as AnthropicMessagesTextBlock[]; - ctx.payload = { ...ctx.payload, system: [...system, IDENTITY_BLOCK] }; + ctx.payload = injectIdentityBlockPayload(ctx.payload); return await run(); }; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts index 3352d64c10..18096f2433 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts @@ -28,18 +28,23 @@ import type { AnthropicMessagesMessage, AnthropicMessagesPayload } from '@floway // References: // - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/metadata_userid.go#L15 +export const synthesizeMetadataUserIdPayload =

>(payload: P, upstreamId: string): P => { + const existing = payload.metadata?.user_id; + if (typeof existing === 'string' && existing.length > 0) return payload; + + const deviceId = deviceIdForUpstream(upstreamId); + const sessionId = sessionIdForPayload(upstreamId, payload); + const userId = JSON.stringify({ device_id: deviceId, account_uuid: '', session_id: sessionId }); + + payload = { ...payload, metadata: { ...payload.metadata, user_id: userId } }; + return payload; +}; + export const synthesizeMetadataUserId = async ( ctx: AnthropicMessagesBoundaryCtx, run: () => Promise, ): Promise => { - const existing = ctx.payload.metadata?.user_id; - if (typeof existing === 'string' && existing.length > 0) return await run(); - - const deviceId = deviceIdForUpstream(ctx.upstreamId); - const sessionId = sessionIdForPayload(ctx.upstreamId, ctx.payload); - const userId = JSON.stringify({ device_id: deviceId, account_uuid: '', session_id: sessionId }); - - ctx.payload = { ...ctx.payload, metadata: { ...ctx.payload.metadata, user_id: userId } }; + ctx.payload = synthesizeMetadataUserIdPayload(ctx.payload, ctx.upstreamId); return await run(); }; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts index 167cfe133e..57598e33c6 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts +++ b/packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts @@ -89,7 +89,7 @@ const FINGERPRINT_INDICES = [4, 7, 20] as const; const encoder = new TextEncoder(); -const extractFirstUserText = (body: AnthropicMessagesPayload): string => { +const extractFirstUserText = (body: Pick): string => { for (const msg of body.messages) { if (msg.role !== 'user') continue; if (typeof msg.content === 'string') return msg.content; @@ -106,7 +106,7 @@ const extractFirstUserText = (body: AnthropicMessagesPayload): string => { // user-message text; positions past the end are filled with 0x30 ('0'). // The output drives `${VERSION}.${FP}` in the billing block; matching // real CC's wire shape is cheap and robust to detector tightening. -export const computeCcVersionFingerprint = (version: string, body: AnthropicMessagesPayload): string => { +export const computeCcVersionFingerprint = (version: string, body: Pick): string => { const utf8 = encoder.encode(extractFirstUserText(body)); const chars = new Uint8Array(FINGERPRINT_INDICES.length); for (let i = 0; i < FINGERPRINT_INDICES.length; i++) { diff --git a/packages/provider-claude-code/src/pipeline-facts.ts b/packages/provider-claude-code/src/pipeline-facts.ts new file mode 100644 index 0000000000..c30e02ff55 --- /dev/null +++ b/packages/provider-claude-code/src/pipeline-facts.ts @@ -0,0 +1,26 @@ +import type { EnsuredAccessToken } from './access-token.ts'; +import type { ClaudeCodeAccountCredential } from './state.ts'; +import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; +import type { Secret } from '@floway-dev/pipeline'; +import type { ProviderModelFacts, ProviderOperationRequest } from '@floway-dev/provider'; + +export type ClaudeCodeRequest = ProviderOperationRequest<'anthropicMessages'>; +export type ClaudeCodeAccountFacts = Omit & { + readonly refreshToken: Secret | null; + readonly accessToken: (Omit, 'token'> & { readonly token: Secret }) | null; +}; +export type ClaudeCodeAccessFacts = Omit & { readonly entry: Omit & { readonly token: Secret } }; +export interface ClaudeCodePreparedRequest extends ClaudeCodeRequest { + 'request.provider.modelKey': string; + 'request.claudeCode.shaped': boolean; +} +export interface ClaudeCodeAuthenticatedRequest extends ClaudeCodePreparedRequest { + 'request.claudeCode.account': ClaudeCodeAccountFacts; + 'request.claudeCode.access': ClaudeCodeAccessFacts; +} +export interface ClaudeCodeHttpRequest extends HttpRequestFacts { + 'request.provider.modelKey': string; + 'request.claudeCode.model': ProviderModelFacts; + 'request.claudeCode.account': ClaudeCodeAccountFacts; + 'request.claudeCode.access': ClaudeCodeAccessFacts; +} diff --git a/packages/provider-claude-code/src/pipelines.ts b/packages/provider-claude-code/src/pipelines.ts new file mode 100644 index 0000000000..9d9ff0d330 --- /dev/null +++ b/packages/provider-claude-code/src/pipelines.ts @@ -0,0 +1,23 @@ +import { authenticateClaudeCode } from './stages/authenticate.ts'; +import { backfillRequiredFields } from './stages/backfill-required-fields.ts'; +import { decodeClaudeCodeResponse } from './stages/decode-response.ts'; +import { hoistUserSystemToMessages } from './stages/hoist-user-system-to-messages.ts'; +import { injectBillingBlock } from './stages/inject-billing-block.ts'; +import { injectDefaultTemplate } from './stages/inject-default-template.ts'; +import { injectIdentityBlock } from './stages/inject-identity-block.ts'; +import { observeClaudeCodeResponse } from './stages/observe-response.ts'; +import { prepareClaudeCodeRequest } from './stages/prepare-request.ts'; +import { recognizeClaudeCodeShape } from './stages/recognize-shape.ts'; +import { retryClaudeCodeAccess } from './stages/retry-access.ts'; +import { synthesizeMetadataUserId } from './stages/synthesize-metadata-user-id.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose } from '@floway-dev/pipeline'; +import { observeProviderCall, type ProviderOperationRequest, type ProviderChatResponse, type ProviderPipelines } from '@floway-dev/provider'; + +export const createClaudeCodePipelines = (upstreamId: string): ProviderPipelines => ({ + anthropicMessages: compose, ProviderChatResponse<'anthropicMessages'>>('claudeCode.anthropicMessages', [ + recognizeClaudeCodeShape, backfillRequiredFields, synthesizeMetadataUserId(upstreamId), hoistUserSystemToMessages, + injectBillingBlock, injectIdentityBlock, injectDefaultTemplate, authenticateClaudeCode(upstreamId), + prepareClaudeCodeRequest, decodeClaudeCodeResponse(upstreamId), retryClaudeCodeAccess(upstreamId), observeClaudeCodeResponse(upstreamId), observeProviderCall, http, + ]), +}); diff --git a/packages/provider-claude-code/src/provider.ts b/packages/provider-claude-code/src/provider.ts index 33e33b3caf..62eeaa13e2 100644 --- a/packages/provider-claude-code/src/provider.ts +++ b/packages/provider-claude-code/src/provider.ts @@ -5,6 +5,7 @@ import { isClaudeCodeShapedRequest } from './detection.ts'; import { callClaudeCodeAnthropicMessages } from './fetch.ts'; import { CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY, type AnthropicMessagesBoundaryCtx } from './interceptors/anthropic-messages/index.ts'; import { buildClaudeCodeCatalog, fetchClaudeCodeModelsList } from './models.ts'; +import { createClaudeCodePipelines } from './pipelines.ts'; import { assertClaudeCodeUpstreamState } from './state.ts'; import { runInterceptors } from '@floway-dev/interceptor'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; @@ -125,7 +126,7 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { disabledPublicModelIds: record.disabledPublicModelIds, modelPrefix: record.modelPrefix, modelsCache: record.modelsCache, - pipelines: {}, + pipelines: createClaudeCodePipelines(record.id), instance, }; }; diff --git a/packages/provider-claude-code/src/stages/authenticate.ts b/packages/provider-claude-code/src/stages/authenticate.ts new file mode 100644 index 0000000000..3c6c75c63c --- /dev/null +++ b/packages/provider-claude-code/src/stages/authenticate.ts @@ -0,0 +1,47 @@ +import { ensureClaudeCodeAccessToken } from '../access-token.ts'; +import { ClaudeCodeOAuthSessionTerminatedError } from '../auth/oauth.ts'; +import { isRateLimitedNow, synthetic429, synthetic503 } from '../fetch.ts'; +import type { ClaudeCodeAuthenticatedRequest, ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { readClaudeCodeUpstreamState } from '../state.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { getProviderRepo, type ProviderChatResponse, type ProviderChatServices } from '@floway-dev/provider'; + +export const authenticateClaudeCode = (upstreamId: string) => defineStage, ProviderChatResponse<'anthropicMessages'>, ProviderChatResponse<'anthropicMessages'>, ProviderChatServices>({ + name: 'authenticateClaudeCode', + through: { + request: { needs: ['request.provider.modelKey', 'request.http.callId'], consumes: [], provides: ['request.claudeCode.account', 'request.claudeCode.access'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.output'] }, + execute: async (facts, next, use) => { + const fresh = await getProviderRepo().upstreams.getById(upstreamId); + if (fresh === null) throw new Error(`Claude Code upstream ${upstreamId} disappeared mid-request`); + const account = readClaudeCodeUpstreamState(fresh.state).accounts[0]; + const accountFacts = { + ...account, refreshToken: account.refreshToken === null ? null : secret(account.refreshToken), + accessToken: account.accessToken === null ? null : { ...account.accessToken, token: secret(account.accessToken.token) }, + }; + const reply = (response: Response) => { + const exchange = takeHttpResponse(response); + return move({ + ...facts, 'request.claudeCode.account': accountFacts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, + 'response.provider.modelKey': facts['request.provider.modelKey'], 'response.provider.called': false, + 'response.provider.previousCalls': [], 'response.provider.output': null, + }); + }; + if (account.state !== 'active') return reply(synthetic503(`Claude Code account is ${account.state}: ${account.stateMessage}`)); + const now = new Date(); + const quota = account.quotaSnapshot === null ? null : account.quotaSnapshot.data; + if (isRateLimitedNow(quota, now)) return reply(synthetic429(quota.reset ? `Claude Code upstream rate-limited until ${quota.reset}` : 'Claude Code upstream rate-limited', quota.reset, now)); + const call = use.httpCall(facts['request.http.callId']); + let access; + try { + access = await ensureClaudeCodeAccessToken({ upstreamId, repo: getProviderRepo().upstreams, fetcher: call.fetcher }); + } catch (error) { + if (error instanceof ClaudeCodeOAuthSessionTerminatedError) return reply(synthetic503(`Claude Code refresh failed: ${error.upstreamMessage}`)); + throw error; + } + return move({ ...await next(move({ ...facts, 'request.claudeCode.account': accountFacts, 'request.claudeCode.access': { ...access, entry: { ...access.entry, token: secret(access.entry.token) } } })) }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/backfill-required-fields.ts b/packages/provider-claude-code/src/stages/backfill-required-fields.ts new file mode 100644 index 0000000000..3845261bae --- /dev/null +++ b/packages/provider-claude-code/src/stages/backfill-required-fields.ts @@ -0,0 +1,10 @@ +import { backfillRequiredFieldsPayload } from '../interceptors/anthropic-messages/backfill-required-fields.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const backfillRequiredFields = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'backfillRequiredFields', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : backfillRequiredFieldsPayload(facts['request.provider.payload'], facts['request.provider.model']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/decode-response.ts b/packages/provider-claude-code/src/stages/decode-response.ts new file mode 100644 index 0000000000..37f7a85475 --- /dev/null +++ b/packages/provider-claude-code/src/stages/decode-response.ts @@ -0,0 +1,41 @@ +import { observedClaudeCodeMessagesStream, STREAM_DIAGNOSTIC_FRAME_DATA_CHARS, STREAM_DIAGNOSTIC_FRAME_LIMIT, type StreamDiagnosticFrame } from '../fetch.ts'; +import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease } from '@floway-dev/pipeline'; +import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; +import { isEventStreamMediaType } from '@floway-dev/protocols/common'; +import type { ProviderChatResponse, ProviderChatServices, ProviderResponse } from '@floway-dev/provider'; + +export const decodeClaudeCodeResponse = (upstreamId: string) => defineStage, ProviderChatServices>({ + name: 'decodeClaudeCodeResponse', + through: { + request: { needs: ['request.provider.modelKey', 'request.http.callId'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.body'], provides: ['response.http.body', 'response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + if (exchange.type === 'transportFailure' || exchange.status < 200 || exchange.status >= 300) return move({ ...back, 'response.provider.output': null }); + const response = exchangeResponse(exchange); + if (exchange.body === null || !isEventStreamMediaType(response.headers.get('content-type'))) { + const text = await response.text(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + let body: unknown; + try { body = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; body = text; } + const snippet = text.length === 0 ? '' : text.length > 1024 ? `${text.slice(0, 1024)}...[truncated]` : text; + return move({ ...back, 'response.provider.output': { status: 502, message: `Upstream returned ${exchange.status} with content-type "${response.headers.get('content-type') ?? 'unknown'}" but stream is required (provider must force stream=true and return text/event-stream when response.ok). Body: ${snippet}`, body } }); + } + const signal = use.httpCall(facts['request.http.callId']).signal; + let rawFrameCount = 0; + const samples: StreamDiagnosticFrame[] = []; + const parsed = use.recordProtocolFrames(parseAnthropicMessagesStream(exchange.body, { + signal, onSseFrame: frame => { + rawFrameCount++; + samples.push({ event: frame.event ?? null, data: frame.data.length > STREAM_DIAGNOSTIC_FRAME_DATA_CHARS ? `${frame.data.slice(0, STREAM_DIAGNOSTIC_FRAME_DATA_CHARS - 3)}...` : frame.data }); + if (samples.length > STREAM_DIAGNOSTIC_FRAME_LIMIT) samples.shift(); + }, + })); + const frames = use.recordProtocolFrames(observedClaudeCodeMessagesStream(parsed, { upstreamId, model: facts['request.provider.modelKey'], headers: response.headers, signal, frames: samples, rawFrameCount: () => rawFrameCount, warn: use.log.warn })); + return move({ ...back, 'response.provider.output': { kind: 'stream' as const, frames } }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts b/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts new file mode 100644 index 0000000000..7d9a709ef5 --- /dev/null +++ b/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts @@ -0,0 +1,10 @@ +import { hoistUserSystemToMessagesPayload } from '../interceptors/anthropic-messages/hoist-user-system-to-messages.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const hoistUserSystemToMessages = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'hoistUserSystemToMessages', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : hoistUserSystemToMessagesPayload(facts['request.provider.payload']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/inject-billing-block.ts b/packages/provider-claude-code/src/stages/inject-billing-block.ts new file mode 100644 index 0000000000..2f2a1e6ce9 --- /dev/null +++ b/packages/provider-claude-code/src/stages/inject-billing-block.ts @@ -0,0 +1,10 @@ +import { injectBillingBlockPayload } from '../interceptors/anthropic-messages/inject-billing-block.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const injectBillingBlock = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'injectBillingBlock', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : injectBillingBlockPayload(facts['request.provider.payload']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/inject-default-template.ts b/packages/provider-claude-code/src/stages/inject-default-template.ts new file mode 100644 index 0000000000..88a42a88c2 --- /dev/null +++ b/packages/provider-claude-code/src/stages/inject-default-template.ts @@ -0,0 +1,10 @@ +import { injectDefaultTemplatePayload } from '../interceptors/anthropic-messages/inject-default-template.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const injectDefaultTemplate = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'injectDefaultTemplate', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : injectDefaultTemplatePayload(facts['request.provider.payload']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/inject-identity-block.ts b/packages/provider-claude-code/src/stages/inject-identity-block.ts new file mode 100644 index 0000000000..45e3194f46 --- /dev/null +++ b/packages/provider-claude-code/src/stages/inject-identity-block.ts @@ -0,0 +1,10 @@ +import { injectIdentityBlockPayload } from '../interceptors/anthropic-messages/inject-identity-block.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const injectIdentityBlock = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'injectIdentityBlock', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : injectIdentityBlockPayload(facts['request.provider.payload']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/observe-response.ts b/packages/provider-claude-code/src/stages/observe-response.ts new file mode 100644 index 0000000000..17c0130e2f --- /dev/null +++ b/packages/provider-claude-code/src/stages/observe-response.ts @@ -0,0 +1,41 @@ +import { detectTerminalSentinel, persistQuotaSnapshot, persistTerminalAccountState } from '../fetch.ts'; +import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; +import { parseClaudeCodeQuotaHeaders } from '../quota.ts'; +import { exchangeResponse, takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defer, defineStage, move, setRelease, type Deferred } from '@floway-dev/pipeline'; +import type { ProviderResponse } from '@floway-dev/provider'; + +type Observed = ProviderResponse & { 'response.claudeCode.background': Deferred; 'response.claudeCode.failureBody': unknown }; + +export const observeClaudeCodeResponse = (upstreamId: string) => defineStage({ + name: 'observeClaudeCodeResponse', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.claudeCode.background', 'response.claudeCode.failureBody'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + const tasks: Promise[] = []; + if (exchange.type === 'transportFailure') return move({ ...back, 'response.claudeCode.background': defer(Promise.all(tasks)), 'response.claudeCode.failureBody': null }); + const response = exchangeResponse(exchange); + if (response.ok || response.status === 429) { + const quota = parseClaudeCodeQuotaHeaders(response.headers); + if (Object.keys(quota.raw).length > 0) tasks.push(persistQuotaSnapshot(upstreamId, quota, use.log.info)); + } + let failureBody: unknown = null; + let retained = exchange; + if (response.status === 400 || response.status === 403) { + const text = await response.text(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + try { failureBody = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; failureBody = text; } + const terminal = detectTerminalSentinel(response.status, text); + if (terminal !== null) tasks.push(persistTerminalAccountState(upstreamId, terminal, response.status === 400 ? 'org_disabled_400_sentinel' : 'org_banned_403_sentinel', response.status, use.log.warn)); + retained = takeHttpResponse(new Response(text, { status: exchange.status, statusText: exchange.statusText, headers: response.headers })); + } + return move({ + ...back, 'response.http.exchange': retained, 'response.http.body': retained.body, + 'response.claudeCode.background': defer(Promise.all(tasks)), 'response.claudeCode.failureBody': failureBody, + }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/prepare-request.ts b/packages/provider-claude-code/src/stages/prepare-request.ts new file mode 100644 index 0000000000..96f048dbd8 --- /dev/null +++ b/packages/provider-claude-code/src/stages/prepare-request.ts @@ -0,0 +1,29 @@ +import { ANTHROPIC_MESSAGES_ENDPOINT } from '../fetch.ts'; +import { pickClaudeCodeHeaders } from '../headers.ts'; +import type { ClaudeCodeAuthenticatedRequest, ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; +import { defineStage, isSecret, move, secret } from '@floway-dev/pipeline'; +import { headersForAnthropicMessagesCall, replaceHttpHeader, type ProviderChatResponse } from '@floway-dev/provider'; + +export const prepareClaudeCodeRequest = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'prepareClaudeCodeRequest', + through: { + request: { needs: ['request.provider.model', 'request.provider.modelKey', 'request.provider.payload', 'request.provider.anthropicBeta', 'request.claudeCode.shaped', 'request.claudeCode.access', 'request.http.headers'], consumes: ['request.provider.model', 'request.provider.payload', 'request.provider.anthropicBeta', 'request.http.headers'], provides: ['request.claudeCode.model', 'request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': payload, 'request.provider.anthropicBeta': beta, ...rest } = facts; + let headers = facts['request.claudeCode.shaped'] + ? headersForAnthropicMessagesCall(facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value]), beta) + : Object.entries(pickClaudeCodeHeaders(facts['request.provider.modelKey'])); + if (!headers.some(([name]) => name.toLowerCase() === 'content-type')) headers = [...headers, ['content-type', 'application/json']]; + return move({ + ...await next(move({ + ...rest, + 'request.claudeCode.model': model, + 'request.http.url': ANTHROPIC_MESSAGES_ENDPOINT, 'request.http.method': 'POST', + 'request.http.headers': replaceHttpHeader(headers, 'authorization', secret(`Bearer ${facts['request.claudeCode.access'].entry.token.reveal()}`)), + 'request.http.body': { ...payload, model: facts['request.provider.modelKey'], stream: true }, 'request.http.encoding': 'json' as const, + })), + }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/recognize-shape.ts b/packages/provider-claude-code/src/stages/recognize-shape.ts new file mode 100644 index 0000000000..6a652c4ae9 --- /dev/null +++ b/packages/provider-claude-code/src/stages/recognize-shape.ts @@ -0,0 +1,18 @@ +import { isClaudeCodeShapedRequest } from '../detection.ts'; +import type { ClaudeCodeProviderData } from '../models.ts'; +import type { ClaudeCodePreparedRequest, ClaudeCodeRequest } from '../pipeline-facts.ts'; +import { defineStage, isSecret, move } from '@floway-dev/pipeline'; +import { headersForAnthropicMessagesCall, type ProviderChatResponse } from '@floway-dev/provider'; + +export const recognizeClaudeCodeShape = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'recognizeClaudeCodeShape', + through: { + request: { needs: ['request.provider.model', 'request.provider.payload', 'request.provider.anthropicBeta', 'request.http.headers'], consumes: [], provides: ['request.provider.modelKey', 'request.claudeCode.shaped'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const headers = facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value]); + const shaped = isClaudeCodeShapedRequest({ headers: new Headers(headersForAnthropicMessagesCall(headers, facts['request.provider.anthropicBeta']).map(([name, value]) => [name, value])), body: { ...facts['request.provider.payload'], model: facts['request.provider.model'].id } }); + return move({ ...await next(move({ ...facts, 'request.claudeCode.shaped': shaped, 'request.provider.modelKey': (facts['request.provider.model'].providerData as ClaudeCodeProviderData).upstreamModelId })) }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/retry-access.ts b/packages/provider-claude-code/src/stages/retry-access.ts new file mode 100644 index 0000000000..78994bc982 --- /dev/null +++ b/packages/provider-claude-code/src/stages/retry-access.ts @@ -0,0 +1,36 @@ +import { ensureClaudeCodeAccessToken, invalidateClaudeCodeAccessToken } from '../access-token.ts'; +import { ClaudeCodeOAuthSessionTerminatedError } from '../auth/oauth.ts'; +import { synthetic503 } from '../fetch.ts'; +import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { getProviderRepo, replaceHttpHeader, type ProviderResponse, type ProviderChatServices } from '@floway-dev/provider'; + +export const retryClaudeCodeAccess = (upstreamId: string) => defineStage({ + name: 'retryClaudeCodeAccess', + through: { + request: { needs: ['request.claudeCode.access', 'request.http.callId', 'request.http.headers', 'request.provider.modelKey'], consumes: ['request.claudeCode.access', 'request.http.headers'], provides: ['request.claudeCode.access', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.http.body', 'response.provider.previousCalls'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.provider.previousCalls'] }, + }, + execute: async (facts, next, use) => { + const first = await next(move({ ...facts })); + const exchange = first['response.http.exchange']; + if (exchange.type !== 'response' || exchange.status !== 401 || facts['request.claudeCode.access'].freshlyMinted) return move({ ...first }); + if (exchange.body !== null) await exchange.body[Symbol.asyncDispose](); + const previous = [...first['response.provider.previousCalls'], { modelKey: first['response.provider.modelKey'] }]; + await invalidateClaudeCodeAccessToken({ upstreamId, repo: getProviderRepo().upstreams }); + let access; + try { + access = await ensureClaudeCodeAccessToken({ upstreamId, repo: getProviderRepo().upstreams, fetcher: use.httpCall(facts['request.http.callId']).fetcher }); + } catch (error) { + if (!(error instanceof ClaudeCodeOAuthSessionTerminatedError)) throw error; + const rejected = takeHttpResponse(synthetic503(`Claude Code refresh failed: ${error.upstreamMessage}`)); + return move({ ...first, 'response.http.exchange': rejected, 'response.http.body': rejected.body, 'response.provider.called': false, 'response.provider.previousCalls': previous }); + } + const retried = await next(move({ + ...facts, 'request.claudeCode.access': { ...access, entry: { ...access.entry, token: secret(access.entry.token) } }, + 'request.http.headers': replaceHttpHeader(facts['request.http.headers'], 'authorization', secret(`Bearer ${access.entry.token}`)), + })); + return move({ ...retried, 'response.provider.previousCalls': previous }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts b/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts new file mode 100644 index 0000000000..2aeadcc943 --- /dev/null +++ b/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts @@ -0,0 +1,10 @@ +import { synthesizeMetadataUserIdPayload } from '../interceptors/anthropic-messages/synthesize-metadata-user-id.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const synthesizeMetadataUserId = (upstreamId: string) => defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'synthesizeMetadataUserId', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : synthesizeMetadataUserIdPayload(facts['request.provider.payload'], upstreamId) })) }), +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa9938e289..b676cd3935 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -530,9 +530,15 @@ importers: packages/provider-claude-code: dependencies: + '@floway-dev/http': + specifier: workspace:* + version: link:../http '@floway-dev/interceptor': specifier: workspace:* version: link:../interceptor + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../protocols From c93ab2f7697259d1a7590e059cc94dd894cfaa2d Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:00:51 +0800 Subject: [PATCH 122/156] feat: route chat calls through provider pipelines Enter typed provider chains with canonical request content, admitted headers, and protocol intent. Preserve upstream status and each actual call observation across credentials, failover, compaction replay, and hosted tools. Reuse the upstream body owner for metering and client completion. Release resumes a partially read meter, closes hosted loops without starting new work, and settles unread client output as failed. Alpha Search dispatch now enters the same child run instead of invoking an opaque model service. Verified all 73 chat test files (1118 cases), gateway typecheck, and scoped lint. Actual Custom HTTP fixtures cover HTTP 201 and unread, partial, completed, and multi-turn hosted lifecycles. --- .../anthropic-messages-count-tokens_test.ts | 16 +-- .../chat/anthropic-messages-pipeline_test.ts | 16 +-- .../chat/anthropic-messages/http_test.ts | 3 +- ...mini-generate-content-count-tokens_test.ts | 6 +- .../gemini-generate-content-pipeline_test.ts | 25 ++-- .../chat/gemini-generate-content/http_test.ts | 3 +- .../openai-chat-completions-pipeline_test.ts | 11 +- .../chat/openai-chat-completions/http_test.ts | 3 +- .../chat/openai-responses-compact_test.ts | 10 +- .../chat/openai-responses-pipeline_test.ts | 11 +- .../openai-responses/hosted-tools/drive.ts | 4 +- .../hosted-tools/image-sub-request_test.ts | 17 ++- .../hosted-tools/lifecycle_test.ts | 86 +++++++++++++ .../hosted-tools/search-sub-request_test.ts | 20 ++- .../openai-responses/hosted-tools_test.ts | 11 +- .../chat/openai-responses/http_test.ts | 5 +- .../data-plane/chat/provider-handoffs_test.ts | 70 ++++++++++ .../data-plane/chat/translated-wires_test.ts | 6 +- .../web-search/alpha-search/upstream_test.ts | 24 +++- .../test-utils/alpha-search-provider.ts | 17 +++ .../test-utils/chat-provider-pipelines.ts | 25 ++++ .../answer-claude-code-probe.ts | 11 +- .../chat/anthropic-messages/call-upstream.ts | 107 ++++------------ .../chat/anthropic-messages/count-tokens.ts | 2 +- .../count-tokens/call-upstream.ts | 89 +++---------- .../count-tokens/prepare-web-search.ts | 5 +- .../chat/anthropic-messages/emit.ts | 9 +- .../chat/anthropic-messages/meter.ts | 1 + .../chat/anthropic-messages/pipeline.ts | 2 +- .../anthropic-messages/web-search-tool.ts | 5 +- .../gateway/src/data-plane/chat/dial-wire.ts | 2 +- .../gemini-generate-content/count-tokens.ts | 2 +- .../measure-via-anthropic-messages.ts | 5 +- .../chat/gemini-generate-content/emit.ts | 9 +- .../chat/gemini-generate-content/pipeline.ts | 2 +- .../gateway/src/data-plane/chat/handoff.ts | 5 +- packages/gateway/src/data-plane/chat/meter.ts | 33 +++-- .../openai-chat-completions/call-upstream.ts | 102 +++------------ .../chat/openai-chat-completions/emit.ts | 9 +- .../chat/openai-chat-completions/meter.ts | 1 + .../chat/openai-chat-completions/pipeline.ts | 2 +- .../answer-websocket-warmup.ts | 1 + .../chat/openai-responses/call-upstream.ts | 120 ++++-------------- .../chat/openai-responses/compact-decrypt.ts | 21 ++- .../chat/openai-responses/compact.ts | 2 +- .../openai-responses/compact/call-upstream.ts | 115 ++++------------- .../chat/openai-responses/compact/dial.ts | 2 +- .../data-plane/chat/openai-responses/emit.ts | 27 ++-- .../chat/openai-responses/hosted-tools.ts | 88 ++++++++----- .../hosted-tools/image-sub-request/prepare.ts | 3 +- .../hosted-tools/image-sub-request/retry.ts | 11 +- .../search-sub-request/execute.ts | 53 ++++++-- .../search-sub-request/services.ts | 7 +- .../hosted-tools/web-search.ts | 37 +++--- .../openai-responses/hydrate-stored-items.ts | 7 +- .../data-plane/chat/openai-responses/meter.ts | 1 + .../chat/openai-responses/pipeline.ts | 2 +- .../chat/openai-responses/websocket.ts | 3 +- .../gateway/src/data-plane/chat/prologue.ts | 4 +- .../src/data-plane/chat/resolve-candidates.ts | 11 +- .../gateway/src/data-plane/chat/services.ts | 4 +- .../data-plane/chat/shared/client-stream.ts | 16 ++- .../data-plane/chat/shared/provider-answer.ts | 26 ++++ .../web-search/alpha-search/execution.ts | 4 +- .../tools/web-search/alpha-search/upstream.ts | 25 +--- 65 files changed, 727 insertions(+), 655 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts create mode 100644 packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts create mode 100644 packages/gateway/__tests__/test-utils/alpha-search-provider.ts create mode 100644 packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts create mode 100644 packages/gateway/src/data-plane/chat/shared/provider-answer.ts diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts index 38d0e4aa92..1f43cffba5 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts @@ -10,11 +10,13 @@ import { anthropicMessagesCountTokensPipeline } from '../../../src/data-plane/ch import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { AliasRules, ModelEndpoints } from '@floway-dev/protocols/common'; import { type FlagId, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; +import type { StubChatProviderCall } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -24,12 +26,7 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal let live: readonly ModelCandidate[] = []; -type CountTokens = ( - model: unknown, - body: unknown, - signal: AbortSignal | undefined, - opts: AnthropicMessagesUpstreamCallOptions, -) => Promise; +type CountTokens = StubChatProviderCall<'anthropicMessagesCountTokens'>; const candidate = ( callAnthropicMessagesCountTokens: CountTokens, @@ -50,7 +47,8 @@ const candidate = ( // rather than proving only that the allowlist did. inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callAnthropicMessagesCountTokens: callAnthropicMessagesCountTokens as never }), + pipelines: stubChatProviderPipelines({ callAnthropicMessagesCountTokens: callAnthropicMessagesCountTokens as never }), + instance: stubProvider(), }, model: stubInternalModel( { @@ -96,6 +94,7 @@ const count = async ( { gateway, background: () => {}, + ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), rememberChatSelection: () => {}, chatPayloadFor: () => affinityPayload, @@ -113,6 +112,7 @@ beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); affinityPayload = payload; initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, } as never); @@ -301,7 +301,7 @@ describe('the messages count-tokens chain', () => { it('fails a dial that never connected over to the next candidate', async () => { const tried: string[] = []; resolves([ - candidate(async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); }, { upstreamId: 'up_dead' }), + candidate(async () => { tried.push('dead'); return { type: 'transportFailure' as const, error: new Error('ECONNREFUSED') }; }, { upstreamId: 'up_dead' }), candidate(async () => { tried.push('alive'); return counted(7); }, { upstreamId: 'up_alive' }), ]); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts index 85bb89f251..b1a86da78b 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts @@ -12,11 +12,13 @@ import { anthropicMessagesServePipeline } from '../../../src/data-plane/chat/ant import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { SseFrame } from '@floway-dev/protocols/common'; import { directFetcher, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; +import type { StubChatProviderCall } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -26,12 +28,7 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal let live: readonly ModelCandidate[] = []; -type CallAnthropicMessages = ( - model: unknown, - body: unknown, - signal: AbortSignal | undefined, - opts: AnthropicMessagesUpstreamCallOptions, -) => Promise>; +type CallAnthropicMessages = StubChatProviderCall<'anthropicMessages'>; const candidate = (callAnthropicMessages: CallAnthropicMessages, upstreamId = 'up_a'): ModelCandidate => { const endpoints = { anthropicMessages: {} }; @@ -42,7 +39,8 @@ const candidate = (callAnthropicMessages: CallAnthropicMessages, upstreamId = 'u // rather than proving only that the allowlist did. inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callAnthropicMessages: callAnthropicMessages as never }), + pipelines: stubChatProviderPipelines({ callAnthropicMessages: callAnthropicMessages as never }), + instance: stubProvider(), }, model: stubInternalModel( { id: 'claude-model', endpoints, providerModels: { [upstreamId]: stubProviderModel({ id: 'claude-model', endpoints }) } }, @@ -120,6 +118,7 @@ const serveWith = async ( { gateway, background: () => {}, + ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), rememberChatSelection: () => {}, chatPayloadFor: () => { asked += 1; return affinityPayload; }, @@ -173,6 +172,7 @@ beforeEach(() => { affinityPayload = payload; asked = 0; initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, } as never); @@ -297,7 +297,7 @@ describe('the messages chain', () => { it('fails a dial that never connected over to the next candidate', async () => { const tried: string[] = []; resolves([ - candidate(async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); }, 'up_dead'), + candidate(async () => { tried.push('dead'); return { type: 'transportFailure' as const, error: new Error('ECONNREFUSED') }; }, 'up_dead'), candidate(async () => { tried.push('alive'); return stream(turn('hi')); }, 'up_alive'), ]); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index edd1d00a51..bc14af8426 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -6,6 +6,7 @@ import { initRepo } from '../../../../src/repo/index.ts'; import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; import { flushBackground } from '../../../test-utils/background-tracker.ts'; +import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; @@ -115,7 +116,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(provider), instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts index cf1eddabde..ecc2ed3ce8 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts @@ -9,6 +9,7 @@ import { geminiGenerateContentCountTokensPipeline } from '../../../src/data-plan import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; @@ -44,7 +45,8 @@ const candidate = ( // rather than proving only that the allowlist did. inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callAnthropicMessagesCountTokens: callAnthropicMessagesCountTokens as never }), + pipelines: stubChatProviderPipelines({ callAnthropicMessagesCountTokens: callAnthropicMessagesCountTokens as never }), + instance: stubProvider(), }, model: stubInternalModel( { @@ -86,6 +88,7 @@ const count = async ( { gateway, background: () => {}, + ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), rememberChatSelection: () => {}, chatPayloadFor: () => request, @@ -102,6 +105,7 @@ const count = async ( beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, } as never); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts index 5992888ff0..5b15b96d18 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts @@ -10,12 +10,14 @@ import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, type GeminiGenerateContentPayload, type GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { directFetcher, type ModelCandidate, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; +import { directFetcher, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; +import type { StubChatProviderCall } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -39,12 +41,7 @@ const resolveAttempt = (selector: { readonly upstreamId: string }): ModelCandida return found; }; -type CallOpenAIChatCompletions = ( - model: unknown, - body: unknown, - signal: AbortSignal | undefined, - opts: UpstreamCallOptions, -) => Promise>; +type CallOpenAIChatCompletions = StubChatProviderCall<'openaiChatCompletions'>; const candidate = (upstream: string, callOpenAIChatCompletions: CallOpenAIChatCompletions): ModelCandidate => { const endpoints = { openaiChatCompletions: {} }; @@ -52,8 +49,8 @@ const candidate = (upstream: string, callOpenAIChatCompletions: CallOpenAIChatCo provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - pipelines: {}, - instance: stubProvider({ callOpenAIChatCompletions: callOpenAIChatCompletions as never }), + pipelines: stubChatProviderPipelines({ callOpenAIChatCompletions: callOpenAIChatCompletions as never }), + instance: stubProvider(), }, model: stubInternalModel({ id: 'gemini-2.5-pro', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'gemini-2.5-pro', endpoints }) } }, upstream), fetcher: directFetcher, @@ -124,6 +121,7 @@ beforeEach(() => { affinityPayload = payload; asked = 0; initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async (row: unknown) => { recorded.usage.push(row); } }, performance: { recordNeutral: async (dims: unknown) => { recorded.performance.push(dims); }, @@ -141,6 +139,7 @@ const serveWith = async (gateway: ReturnType, facts: { gateway, background: () => {}, + ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), rememberChatSelection: () => {}, chatPayloadFor: () => { asked += 1; return affinityPayload; }, @@ -322,13 +321,14 @@ describe('the Gemini generateContent pipeline', () => { // An upstream that was called and reported nothing, which is a different statement from // reporting zero. expect(facts['response.usage.billable']).toEqual([ + { identity: { model: 'gemini-2.5-pro', upstream: 'up_a', modelKey: 'k', pricing: null }, quantities: {} }, { identity: { model: 'gemini-2.5-pro', upstream: 'up_b', modelKey: 'k', pricing: null }, quantities: {} }, ]); await drain(); }); it('turns a dial that threw into a failure value rather than a thrown run', async () => { - resolves([candidate('up_a', () => Promise.reject(new Error('socket hang up')))]); + resolves([candidate('up_a', async () => ({ type: 'transportFailure', error: new Error('socket hang up') }))]); const { facts, drain } = await serve(entryFacts()); @@ -357,8 +357,7 @@ describe('the Gemini generateContent pipeline', () => { // The wire below always streams, whatever the client asked for, so this family's numbers // always arrive with the last chunk — which is after the run has answered. Settling in the - // stage as well would write the row twice, so the pipeline hands the numbers up as a promise - // and the epilogue is what writes them. + // settlement stage registers that promise once and writes the measured result at completion. it('defers settlement to the promise it hands up, metered on the dialect the upstream spoke', async () => { resolves([candidate('up_a', async () => streamed(turn()))]); @@ -366,7 +365,7 @@ describe('the Gemini generateContent pipeline', () => { await collect(facts['response.chat.geminiGenerateContent.rendered']); await drain(); - expect(recorded.usage).toHaveLength(0); + expect(recorded.usage).toHaveLength(1); const usage = facts['response.chat.geminiGenerateContent.streamedUsage']; expect(usage).not.toBeNull(); const outcome = await usage!; diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts index 478e5d6ddb..f5d0245067 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts @@ -5,6 +5,7 @@ import type { AuthVars } from '../../../../src/middleware/auth.ts'; import { initRepo } from '../../../../src/repo/index.ts'; import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; +import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; @@ -118,7 +119,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(provider), instance: provider, }, model: stubInternalModel({ endpoints }, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts index e6d7e3eaaa..fef7b7367a 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts @@ -10,11 +10,13 @@ import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; import { type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; +import type { StubChatProviderCall } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -25,7 +27,7 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal let live: readonly ModelCandidate[] = []; const candidate = ( - callOpenAIChatCompletions: (model: unknown, body: unknown) => Promise>, + callOpenAIChatCompletions: StubChatProviderCall<'openaiChatCompletions'>, upstreamId = 'up_a', flags: readonly FlagId[] = [], ): ModelCandidate => { @@ -34,7 +36,8 @@ const candidate = ( provider: { upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callOpenAIChatCompletions }), + pipelines: stubChatProviderPipelines({ callOpenAIChatCompletions }), + instance: stubProvider(), }, model: stubInternalModel( { @@ -110,6 +113,7 @@ const serveWith = async ( { gateway, background: () => {}, + ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), rememberChatSelection: () => {}, chatPayloadFor: () => affinityPayload, @@ -127,6 +131,7 @@ const serveWith = async ( beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, } as never); @@ -451,7 +456,7 @@ describe('the chat completions chain', () => { it('fails a dial that never connected over to the next candidate', async () => { const tried: string[] = []; resolves([ - candidate(async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); }, 'up_dead'), + candidate(async () => { tried.push('dead'); return { type: 'transportFailure' as const, error: new Error('ECONNREFUSED') }; }, 'up_dead'), candidate(async () => { tried.push('alive'); return stream(chunk('hi')); }, 'up_alive'), ]); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts index 730bbe7d2d..2d23675ae6 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts @@ -8,6 +8,7 @@ import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { installDumpStubs } from '../../../dump/test-fixtures.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; import { flushBackground } from '../../../test-utils/background-tracker.ts'; +import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { type ModelCandidate, directFetcher, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; @@ -112,7 +113,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(provider), instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts index 6be8daf596..e2ab1ce7b5 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts @@ -15,6 +15,7 @@ import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candid import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; @@ -44,7 +45,8 @@ const candidate = ( provider: { upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider(calls as never), + pipelines: stubChatProviderPipelines(calls as never), + instance: stubProvider(), }, model: stubInternalModel( { @@ -163,6 +165,7 @@ const compact = async (request: CanonicalOpenAIResponsesPayload = payload, signa { gateway, background: () => {}, + ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), rememberChatSelection: () => {}, chatPayloadFor: () => request, @@ -197,6 +200,7 @@ beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); settlements.length = 0; initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, openaiResponsesSnapshots: { lookup: async () => null, put: async () => {} }, @@ -438,8 +442,8 @@ describe('native compaction decryption', () => { }, { enabledFlags: decryptFlags })]); const { facts } = await compact({ ...payload, instructions: 'original instructions', tools: [{ type: 'function', name: 'f' }] }); expect(calls.map(call => call.action)).toEqual(['compact', 'generate', 'generate']); - expect(calls[1].model).toBe(calls[0].model); - expect(calls[2].model).toBe(calls[0].model); + expect(calls[1].model).toStrictEqual(calls[0].model); + expect(calls[2].model).toStrictEqual(calls[0].model); for (const call of calls.slice(1)) { expect(call.body).not.toHaveProperty('instructions'); expect(call.body).not.toHaveProperty('tools'); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts index 960eab3387..f89e386ad4 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -18,12 +18,14 @@ import type { StreamOutcome } from '../../../src/data-plane/pipeline/serve.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run, type Deferred } from '@floway-dev/pipeline'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ModelEndpoints, SseFrame } from '@floway-dev/protocols/common'; import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesCompactionResult, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; +import type { StubChatProviderCall } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -35,7 +37,7 @@ let live: readonly ModelCandidate[] = []; const candidate = ( calls: { - callOpenAIResponses?: (model: unknown, body: unknown, action: unknown) => Promise; + callOpenAIResponses?: StubChatProviderCall<'openaiResponses'>; callAnthropicMessages?: (model: unknown, body: unknown) => Promise>; }, overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, @@ -47,7 +49,8 @@ const candidate = ( provider: { upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider(calls as never), + pipelines: stubChatProviderPipelines(calls as never), + instance: stubProvider(), }, model: stubInternalModel( { @@ -169,6 +172,7 @@ const serveWith = async ( { gateway, background: () => {}, + ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), rememberChatSelection: () => {}, chatPayloadFor: () => { asked += 1; return affinityPayload; }, @@ -243,6 +247,7 @@ beforeEach(() => { asked = 0; vi.mocked(enumerateModelCandidates).mockReset(); initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, } as never); @@ -556,7 +561,7 @@ describe('the responses chain', () => { it('fails a dial that never connected over to the next candidate', async () => { const tried: string[] = []; resolves([ - candidate({ callOpenAIResponses: async () => { tried.push('dead'); throw new Error('ECONNREFUSED'); } }, { upstreamId: 'up_dead' }), + candidate({ callOpenAIResponses: async () => { tried.push('dead'); return { type: 'transportFailure' as const, error: new Error('ECONNREFUSED') }; } }, { upstreamId: 'up_dead' }), candidate({ callOpenAIResponses: async () => { tried.push('alive'); return stream(completed('hi')); } }, { upstreamId: 'up_alive' }), ]); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts index 91a3c7f93e..0a6bf0913c 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts @@ -74,7 +74,7 @@ export const driveHostedToolStage = (registrations: readonly HostedToolRegistrat ): Promise | null }> => { const ending = defineStage, Record>({ name: 'scriptedDial', - return: { provides: [ANSWER, STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'] }, + return: { provides: [ANSWER, STREAMED_USAGE, 'response.usage.billable', 'response.http.headers', 'response.http.body'] }, execute: async facts => { invocation.payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; const result = await dial(); @@ -88,6 +88,7 @@ export const driveHostedToolStage = (registrations: readonly HostedToolRegistrat ? [{ identity: result.modelIdentity, quantities: {} }] : [], 'response.http.headers': [], + 'response.http.body': null, }); }, }); @@ -102,6 +103,7 @@ export const driveHostedToolStage = (registrations: readonly HostedToolRegistrat const outcome = await run(chain, move({ 'request.chat.openaiResponses': invocation.payload, + 'serve.usage.prior': [], 'route.attempt': { candidateId: 0, upstreamId: 'up_test', modelId: invocation.payload.model, flags: [] }, 'ingress.http.headers': [...invocation.headers], }), { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts index 72c936d173..a284630668 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts @@ -11,6 +11,7 @@ import { eventsOf, installDumpStubs } from '../../../../dump/test-fixtures.ts'; import { InMemoryRepo } from '../../../../repo/memory.ts'; import { flushBackground, trackBackground } from '../../../../test-utils/background-tracker.ts'; import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { stubProviderPipeline } from '../../../../test-utils/provider-pipeline.ts'; import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; import { stubModelCandidate } from '@floway-dev/test-utils'; @@ -24,14 +25,22 @@ for (const action of ['generate', 'edit'] as const) for (const consume of [false const dumps = installDumpStubs(initDumpStore, initDumpBroker); const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); - const candidate = stubModelCandidate({ model: { id: 'image-model', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } } }); + const plainCandidate = stubModelCandidate({ model: { id: 'image-model', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } } }); const prefix = action === 'edit' ? 'image_edit' : 'image_generation'; const response = () => new Response([ { type: `${prefix}.partial_image`, partial_image_index: 0, b64_json: 'preview' }, { type: `${prefix}.completed`, b64_json: 'pixels', usage: { input_tokens: 7, output_tokens: 3 } }, ].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } }); - const generate = vi.spyOn(candidate.provider.instance, 'callOpenAIImagesGenerations').mockImplementation(async () => ({ response: response(), modelKey: 'backend-image' })); - const edit = vi.spyOn(candidate.provider.instance, 'callOpenAIImagesEdits').mockImplementation(async () => ({ response: response(), modelKey: 'backend-image' })); + const generate = vi.fn(async () => ({ response: response(), modelKey: 'backend-image' })); + const edit = vi.fn(async () => ({ response: response(), modelKey: 'backend-image' })); + const candidate = { + ...plainCandidate, provider: { + ...plainCandidate.provider, pipelines: { + openaiImagesGenerations: stubProviderPipeline('openaiImagesGenerations', generate), + openaiImagesEdits: stubProviderPipeline('openaiImagesEdits', edit), + }, + }, + }; vi.spyOn(resolution, 'enumerateModelCandidates').mockResolvedValue({ candidates: [candidate], sawModel: true, failedUpstreams: [] }); const request: ImageGenerationRequest = { prompt: 'draw a tree', action, config: { model: 'image-model', action, quality: 'high', partial_images: 1 }, @@ -59,7 +68,7 @@ for (const action of ['generate', 'edit'] as const) for (const consume of [false expect(prepared).toMatchObject({ operation: action === 'edit' ? 'edits' : 'generations', parameters: { prompt: 'draw a tree', n: 1, quality: 'high', stream: true, partial_images: 1 } }); if (action === 'edit') expect(prepared).toMatchObject({ images: [{ kind: 'file', file: { fileName: 'image_0.png', mediaType: 'image/png', bytes: { bytes: 'AQID' } } }] }); const entered = eventsOf(record).filter(event => event.type === 'stage.entered').map(event => event.name); - expect(entered).toEqual(['emitHostedImageGeneration', 'prepareHostedImageGeneration', 'writeSettlement', 'resolveCandidates', 'failover', 'retryRateLimitedImages', 'callOpenAIImagesUpstream']); + expect(entered).toEqual(['writeSettlement', 'emitHostedImageGeneration', 'prepareHostedImageGeneration', 'resolveCandidates', 'failover', 'retryRateLimitedImages', 'callOpenAIImagesUpstream', 'stubHttp']); const frames = decoded.flatMap(event => event?.frames ?? []); if (consume) expect(frames).toContainEqual({ type: 'event', event: { item: { type: 'image_generation_call', status: 'completed', action, result: 'pixels', revised_prompt: 'draw a tree' }, endEvents: [{ type: 'response.image_generation_call.completed' }] } }); else expect(frames).toEqual([]); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts new file mode 100644 index 0000000000..bbc444f948 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts @@ -0,0 +1,86 @@ +import { expect, test } from 'vitest'; + +import { emitOpenAIResponses } from '../../../../../src/data-plane/chat/openai-responses/emit.ts'; +import type { HostedToolRegistration } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/types.ts'; +import { hostedTools } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools.ts'; +import { openaiResponsesWire } from '../../../../../src/data-plane/chat/openai-responses/wire.ts'; +import { prologueFor, type StreamOutcome } from '../../../../../src/data-plane/pipeline/serve.ts'; +import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { isReplayableBody } from '@floway-dev/http'; +import { compose, move, run, type Deferred } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { ModelCandidate } from '@floway-dev/provider'; +import { createCustomPipelines } from '@floway-dev/provider-custom'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +const key = 'response.chat.openaiResponses.streamedUsage'; +const payload: CanonicalOpenAIResponsesPayload = { model: 'model', input: [], tools: [{ type: 'web_search' }] }; +const shell = (status: OpenAIResponsesResult['status'], input: number, output: number): OpenAIResponsesResult => ({ id: 'upstream', object: 'response', model: 'model', output: [], status, error: null, incomplete_details: null, usage: { input_tokens: input, output_tokens: output, total_tokens: input + output } }); +const first: OpenAIResponsesStreamEvent[] = [ + { type: 'response.created', response: shell('in_progress', 0, 0) }, + { type: 'response.output_item.added', output_index: 0, item: { type: 'function_call', id: 'tool', call_id: 'call', name: 'test_search', arguments: '', status: 'in_progress' } }, + { type: 'response.output_item.done', output_index: 0, item: { type: 'function_call', id: 'tool', call_id: 'call', name: 'test_search', arguments: '{}', status: 'completed' } }, + { type: 'response.completed', response: shell('completed', 3, 2) }, +]; +const second: OpenAIResponsesStreamEvent[] = [ + { type: 'response.created', response: shell('in_progress', 0, 0) }, + { type: 'response.output_text.delta', output_index: 0, item_id: 'message', content_index: 0, delta: 'partial' }, + { type: 'response.completed', response: shell('completed', 4, 3) }, +]; + +for (const client of ['unread', 'partial-before-tool', 'partial-second', 'complete'] as const) test(`hosted ReAct settles actual model calls after ${client} output`, async () => { + let calls = 0; + let tools = 0; + const registration: HostedToolRegistration = () => ({ + type: 'active', baseToolName: 'test_search', hosted: { + hostedTypes: ['web_search'], + canonicalize: tool => tool.type === 'web_search' ? tool : undefined, + buildFunctionTool: (_tool, name) => ({ type: 'function', name, description: 'Search', parameters: { type: 'object' }, strict: false }), + dispatcher: () => { + tools += 1; + return [{ id: 'search-item', startItem: { type: 'web_search_call', status: 'in_progress' }, startEvents: [], async *run() { return { item: { type: 'web_search_call', status: 'completed' }, endEvents: [] }; } }]; + }, + }, + }); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + const base = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }); + const plain = stubModelCandidate(); + const candidate: ModelCandidate = { + ...plain, + provider: { ...plain.provider, pipelines: createCustomPipelines({ baseUrl: 'https://custom.example', authStyle: 'none', endpoints: {}, ingressHeadersRules: [], modelsFetch: { enabled: false }, models: [] }) }, + fetcher: async (_url, init) => { + if (!isReplayableBody(init.body)) throw new Error('Expected JSON HTTP request'); + expect(await new Response(init.body.open()).json()).toMatchObject({ stream: true }); + calls += 1; + return new Response((calls === 1 ? first : second).map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } }); + }, + }; + const [selector] = base.services.rememberCandidates([candidate]); + const executed = await run(compose, Record>('hostedLifecycle', [ + emitOpenAIResponses(payload, 'sse'), hostedTools([registration], { streamedUsage: key, targetOf: () => 'openaiResponses' }), ...openaiResponsesWire(key), + ]), move({ 'request.chat.openaiResponses': payload, 'route.attempt': selector, 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'openaiResponses', 'ingress.chat.openaiResponses.wantsStream': true, 'serve.usage.prior': [] }), { + ...base.services, gateway, recordProtocolFrames: (frames: AsyncIterable) => frames, selectAffinity: () => {}, + }); + const rendered = executed.facts['response.chat.openaiResponses.rendered'] as AsyncIterable; + if (client !== 'unread') { + const iterator = rendered[Symbol.asyncIterator](); + if (client === 'complete') { + let step = await iterator.next(); + while (!step.done) step = await iterator.next(); + } else if (client === 'partial-before-tool') await iterator.next(); + else { + let step = await iterator.next(); + while (!step.done && !(calls === 2 && step.value.data.includes('partial'))) step = await iterator.next(); + expect(step.done).toBe(false); + } + if (client !== 'complete') await iterator.return?.(); + } + await executed.drain(); + const outcome = await (executed.facts[key] as Deferred); + const repeated = client === 'partial-second' || client === 'complete'; + expect(calls).toBe(repeated ? 2 : 1); + expect(tools).toBe(repeated ? 1 : 0); + expect(outcome.failed).toBe(client !== 'complete'); + expect(outcome.billable.map(entity => [entity.quantities.input_tokens, entity.quantities.output_tokens])).toEqual(repeated ? [['3', '2'], ['4', '3']] : [['3', '2']]); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts index 069478425e..36a9243d3e 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts @@ -4,8 +4,11 @@ import type { WebSearchRequest } from '../../../../../src/data-plane/chat/openai import { runWebSearchSubRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts'; import { initDumpBroker, initDumpStore } from '../../../../../src/dump/registry.ts'; import { openRunDump } from '../../../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../../../src/repo/index.ts'; import type { ApiKey } from '../../../../../src/repo/types.ts'; import { eventsOf, installDumpStubs } from '../../../../dump/test-fixtures.ts'; +import { InMemoryRepo } from '../../../../repo/memory.ts'; +import { stubAlphaSearchCandidate } from '../../../../test-utils/alpha-search-provider.ts'; import { flushBackground, trackBackground } from '../../../../test-utils/background-tracker.ts'; import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; @@ -19,6 +22,7 @@ const request: WebSearchRequest = { for (const failed of [false, true]) { test(`web search records backend content before dispatch and closes its ${failed ? 'failed' : 'successful'} child run`, async () => { + initRepo(new InMemoryRepo()); const dumps = installDumpStubs(initDumpStore, initDumpBroker); const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); @@ -26,13 +30,14 @@ for (const failed of [false, true]) { let dispatched = false; const invocation = runWebSearchSubRequest(parent, request, { session: { pageCache: new Map(), getProvider: () => { throw new Error('local provider must not dispatch in alpha mode'); }, apiKeyId: apiKey.id }, - executeAlpha: async (content, action) => { - dispatched = true; - expect(content).toEqual(request); - expect(Object.isFrozen(content.commands)).toBe(true); - expect(action).toEqual({ type: 'search', query: 'pipeline streams', queries: ['pipeline streams'] }); - if (failed) throw fault; - return { action, results: [], outputText: 'documentation' }; + alpha: { + sessionId: 'search-session', candidate: Promise.resolve(stubAlphaSearchCandidate(async body => { + dispatched = true; + expect(body).toEqual({ id: 'search-session', commands: request.commands, settings: request.settings, input: request.input }); + expect(Object.isFrozen(body.commands)).toBe(true); + if (failed) throw fault; + return new Response(JSON.stringify({ output: 'documentation' }), { headers: { 'content-type': 'application/json' } }); + })), }, }); if (failed) await expect(invocation).rejects.toBe(fault); @@ -47,6 +52,7 @@ for (const failed of [false, true]) { const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); const entered = decoded.find(event => event?.facts && 'request.webSearch.canonical' in event.facts); expect(entered?.facts?.['request.webSearch.canonical']).toEqual(request); + expect(decoded).toContainEqual(expect.objectContaining({ facts: expect.objectContaining({ 'request.provider.payload': { id: 'search-session', commands: request.commands, settings: request.settings, input: request.input } }) })); expect(new TextDecoder().decode(record.events)).not.toContain('searchCall'); }); } diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts index 8eb9b9f54d..744391ed0f 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts @@ -7,7 +7,7 @@ import { type InterceptedFunctionCall, type HostedToolResultSlot, type TurnSumma import { FUNCTION_TOOL_NAME, webSearchHostedTool } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/web-search.ts'; import { createNonOpenAIResponsesSourceStore } from '../../../../src/data-plane/chat/openai-responses/items/store.ts'; import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; -import { resolveAlphaSearchDispatcher } from '../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; +import { resolveAlphaSearchCandidate } from '../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; import type { AlphaSearchDispatcher } from '../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; import { resolveConfiguredWebSearchProvider } from '../../../../src/data-plane/tools/web-search/provider.ts'; import type { @@ -24,6 +24,7 @@ import { openRunDump } from '../../../../src/dump/run-sink.ts'; import { getRepo, initRepo } from '../../../../src/repo/index.ts'; import { eventsOf, installDumpStubs } from '../../../dump/test-fixtures.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; +import { stubAlphaSearchCandidate } from '../../../test-utils/alpha-search-provider.ts'; import { flushBackground } from '../../../test-utils/background-tracker.ts'; import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; @@ -282,7 +283,7 @@ interface DepsOverrides { } const mockResolveConfigured = vi.mocked(resolveConfiguredWebSearchProvider); -const mockResolveAlpha = vi.mocked(resolveAlphaSearchDispatcher); +const mockResolveAlpha = vi.mocked(resolveAlphaSearchCandidate); // Seed a per-test InMemoryRepo and a default tavily search config so // `loadWebSearchConfig()` returns a non-default value. The actual provider @@ -4059,7 +4060,7 @@ test('responses target with OpenAI passthrough forwards the complete alpha-searc output: 'alpha output', results: [{ type: 'text_result', url: 'https://example.com', title: 'Example', snippet: 'alpha snippet' }], }), { status: 200, headers: { 'content-type': 'application/json' } })); - mockResolveAlpha.mockResolvedValue(call); + mockResolveAlpha.mockResolvedValue(stubAlphaSearchCandidate(call)); const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']), @@ -4121,13 +4122,13 @@ test('local and cascaded Floway unsupported commands produce the same agent-visi jina: { apiKey: '' }, passthroughOpenAiSearch: { enabled: true, upstreamId: 'up_floway', model: 'gpt-search' }, } satisfies WebSearchConfig); - mockResolveAlpha.mockResolvedValue(async () => new Response(JSON.stringify({ + mockResolveAlpha.mockResolvedValue(stubAlphaSearchCandidate(async () => new Response(JSON.stringify({ error: { type: 'internal_error', name: 'UnsupportedLocalWebSearchFeatureError', message: 'The configured web search provider does not implement OpenAI search feature `commands.image_query`.', }, - }), { status: 500, headers: { 'content-type': 'application/json' } })); + }), { status: 500, headers: { 'content-type': 'application/json' } }))); const cascadedMessage = await runUnsupported(); assertEquals(localMessage, expected); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts index 7cf31a1327..0541b2d3dc 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts @@ -7,6 +7,7 @@ import type { AuthVars } from '../../../../src/middleware/auth.ts'; import { initRepo } from '../../../../src/repo/index.ts'; import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; +import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; @@ -140,7 +141,7 @@ const makeCandidate = (overrides: { inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, pipelines: {}, + modelsCache: null, pipelines: stubChatProviderPipelines(provider), instance: provider, }, model: stubInternalModel({ @@ -830,7 +831,7 @@ const translatedCustomCandidate = ( }; }, }); - return { ...candidate, provider: { ...candidate.provider, instance } }; + return { ...candidate, provider: { ...candidate.provider, instance, pipelines: stubChatProviderPipelines(instance) } }; }; for (const target of ['openaiChatCompletions', 'anthropicMessages'] as const) { diff --git a/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts b/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts new file mode 100644 index 0000000000..74acc8c95e --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts @@ -0,0 +1,70 @@ +import { expect, test } from 'vitest'; + +import { emitOpenAIChatCompletions } from '../../../src/data-plane/chat/openai-chat-completions/emit.ts'; +import { openaiChatCompletionsWire } from '../../../src/data-plane/chat/openai-chat-completions/wire.ts'; +import { prologueFor } from '../../../src/data-plane/pipeline/serve.ts'; +import type { StreamOutcome } from '../../../src/data-plane/pipeline/serve.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { isReplayableBody } from '@floway-dev/http'; +import { compose, move, run, type Deferred, type Event } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import type { ModelCandidate } from '@floway-dev/provider'; +import { createCustomPipelines } from '@floway-dev/provider-custom'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +const config = { baseUrl: 'https://custom.example', authStyle: 'none' as const, endpoints: {}, ingressHeadersRules: [], modelsFetch: { enabled: false }, models: [] }; +const first = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'wire-model', choices: [{ index: 0, delta: { content: 'partial' }, finish_reason: null }] }; +const usage = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'wire-model', choices: [], usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } }; + +for (const client of ['unread', 'partial', 'complete'] as const) test(`typed provider handoff preserves HTTP 201 and drains one reading after ${client} client output`, async () => { + const gateway = mockChatGatewayCtx({ wantsStream: true }); + const base = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }); + const plain = stubModelCandidate({ model: { id: 'alias', endpoints: { openaiChatCompletions: {} } }, providerData: 'wire-model' }); + let calls = 0; + const candidate: ModelCandidate = { + ...plain, + provider: { ...plain.provider, pipelines: createCustomPipelines(config) }, + fetcher: async (url, init) => { + expect(url).toBe('https://custom.example/v1/chat/completions'); + if (!isReplayableBody(init.body)) throw new Error('JSON HTTP content must be replayable'); + calls += 1; + expect(await new Response(init.body.open()).json()).toMatchObject({ model: 'wire-model', stream: true, messages: [{ role: 'user', content: 'hello' }], stream_options: { include_usage: true } }); + return new Response([first, usage].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { status: 201, headers: { 'content-type': 'text/event-stream', 'x-upstream': 'kept' } }); + }, + }; + const [selector] = base.services.rememberCandidates([candidate]); + const events: Event[] = []; + const executed = await run(compose, Record>('typedChat', [emitOpenAIChatCompletions, ...openaiChatCompletionsWire('response.chat.openaiChatCompletions.streamedUsage')]), move({ + 'ingress.chat.openaiChatCompletions.wantsStream': true, + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false, + 'ingress.chat.sourceProtocol': 'openaiChatCompletions', + 'ingress.http.headers': [], + 'request.chat.openaiChatCompletions': { model: 'alias', messages: [{ role: 'user', content: 'hello' }] }, + 'route.attempt': selector, + }), { + ...base.services, gateway, + recordProtocolFrames: (frames: AsyncIterable) => frames, + selectAffinity: () => {}, + dump: (event: Event) => { events.push(event); }, + }); + expect(executed.facts['response.http.status']).toBe(201); + expect(executed.facts['response.http.headers']).toContainEqual(['x-upstream', 'kept']); + const frames = executed.facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable; + if (client === 'partial') { + const iterator = frames[Symbol.asyncIterator](); + expect((await iterator.next()).value?.data).toContain('partial'); + await iterator.return?.(); + } else if (client === 'complete') { + const sent: SseFrame[] = []; + for await (const frame of frames) sent.push(frame); + expect(sent).toHaveLength(1); + } + await executed.drain(); + const outcome = await (executed.facts['response.chat.openaiChatCompletions.streamedUsage'] as Deferred); + expect(outcome.failed).toBe(client !== 'complete'); + expect(outcome.billable).toHaveLength(1); + expect(outcome.billable[0].quantities).toMatchObject({ input_tokens: '10', output_tokens: '5' }); + expect(calls).toBe(1); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.entered', name: 'http' })); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.entered', facts: expect.objectContaining({ 'request.provider.payload': expect.objectContaining({ messages: [{ role: 'user', content: 'hello' }] }), 'request.provider.model': expect.objectContaining({ enabledFlags: [] }) }) })); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 0c4fc5a4be..95fc615a25 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -23,6 +23,7 @@ import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/opena import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { compose, move, run, type Pipeline } from '@floway-dev/pipeline'; import { PROMPT_TOO_LONG_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesResult, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; @@ -65,7 +66,8 @@ const candidate = ( provider: { upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider(calls as never), + pipelines: stubChatProviderPipelines(calls as never), + instance: stubProvider(), }, model: stubInternalModel( { @@ -173,6 +175,7 @@ const serve = async ( return await run(pipeline, move(facts) as never, { gateway, background: () => {}, + ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), rememberChatSelection: () => {}, chatPayloadFor: () => attemptPayload, @@ -251,6 +254,7 @@ const geminiGenerateContentText = (rendered: unknown): string | undefined => beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, } as never); diff --git a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts b/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts index 8b5f6a182f..bc323639f7 100644 --- a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts +++ b/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts @@ -1,5 +1,9 @@ import { test, vi } from 'vitest'; +import { providerEntry } from '../../../../../src/data-plane/pipeline/provider-entry.ts'; +import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { stubProviderPipeline } from '../../../../test-utils/provider-pipeline.ts'; +import { run } from '@floway-dev/pipeline'; import type { InboundHeaderMatcher, ModelCandidate } from '@floway-dev/provider'; import { assertEquals, assertExists, stubModelCandidate, stubProvider } from '@floway-dev/test-utils'; @@ -16,7 +20,7 @@ vi.mock('../../../../../src/data-plane/providers/resolution.ts', async importOri }; }); -const { resolveAlphaSearchDispatcher } = await import('../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'); +const { resolveAlphaSearchCandidate } = await import('../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'); const dispatcherFor = async (kind: 'codex' | 'custom', inboundHeaderAllowlist: readonly InboundHeaderMatcher[] = []) => { let observedHeaders: Headers | undefined; @@ -26,6 +30,12 @@ const dispatcherFor = async (kind: 'codex' | 'custom', inboundHeaderAllowlist: r upstreamId: 'search-upstream', kind, inboundHeaderAllowlist, + pipelines: { + alphaSearch: stubProviderPipeline('alphaSearch', async (_model, _body, _signal, opts) => { + observedHeaders = opts.headers; + return { response: new Response('{}'), modelKey: 'search-model' }; + }), + }, instance: stubProvider({ callAlphaSearch: async (_model, _body, _signal, opts) => { observedHeaders = opts.headers; @@ -34,13 +44,21 @@ const dispatcherFor = async (kind: 'codex' | 'custom', inboundHeaderAllowlist: r }), }; resolvedCandidate = stubModelCandidate({ provider }); - const dispatcher = await resolveAlphaSearchDispatcher({ + const dispatcher = await resolveAlphaSearchCandidate({ config: { upstreamId: provider.upstreamId, model: 'search-model' }, upstreamIds: null, scheduler: promise => { void promise; }, runtimeLocation: 'TEST', }); - return { dispatcher, observedHeaders: () => observedHeaders }; + return { + dispatcher: async (body: Record, _signal: AbortSignal | undefined, headers: Headers) => { + const pipeline = dispatcher.provider.pipelines.alphaSearch; + if (pipeline === undefined) throw new Error('Missing selected Alpha Search pipeline'); + const entry = providerEntry({ 'route.attempt': { candidateId: 0, upstreamId: 'search-upstream', modelId: 'search-model', flags: [] }, 'ingress.http.headers': [...headers] }, dispatcher, body); + const executed = await run(pipeline, entry, { gateway: mockGatewayCtx() }); + await executed.drain(); + }, observedHeaders: () => observedHeaders, + }; }; test('Codex Alpha Search receives only its declared turn metadata', async () => { diff --git a/packages/gateway/__tests__/test-utils/alpha-search-provider.ts b/packages/gateway/__tests__/test-utils/alpha-search-provider.ts new file mode 100644 index 0000000000..bc37eac7b6 --- /dev/null +++ b/packages/gateway/__tests__/test-utils/alpha-search-provider.ts @@ -0,0 +1,17 @@ +import { stubProviderPipeline } from './provider-pipeline.ts'; +import type { AlphaSearchDispatcher } from '../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; +import type { ModelCandidate } from '@floway-dev/provider'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +export const stubAlphaSearchCandidate = (dispatch: AlphaSearchDispatcher): ModelCandidate => { + const base = stubModelCandidate(); + return { + ...base, provider: { + ...base.provider, pipelines: { + alphaSearch: stubProviderPipeline('alphaSearch', async (_model, payload, signal, options) => ({ + response: await dispatch(payload, signal, options.headers), modelKey: 'search-model', + })), + }, + }, + }; +}; diff --git a/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts b/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts new file mode 100644 index 0000000000..d9cbf35e7d --- /dev/null +++ b/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts @@ -0,0 +1,25 @@ +import type { GatewayCtx } from '../../src/data-plane/shared/gateway-ctx.ts'; +import type { ProviderPipelines, ProviderChatServices } from '@floway-dev/provider'; +import { noopUpstreamCallOptions, stubChatProviderPipeline, type StubChatProviderCall } from '@floway-dev/test-utils'; + +type Calls = Partial<{ + callOpenAIChatCompletions: StubChatProviderCall<'openaiChatCompletions'>; + callOpenAIResponses: StubChatProviderCall<'openaiResponses'>; + callAnthropicMessages: StubChatProviderCall<'anthropicMessages'>; + callAnthropicMessagesCountTokens: StubChatProviderCall<'anthropicMessagesCountTokens'>; +}>; + +export const stubChatProviderPipelines = (calls: Calls): ProviderPipelines => ({ + ...(calls.callOpenAIChatCompletions === undefined ? {} : { openaiChatCompletions: stubChatProviderPipeline('openaiChatCompletions', calls.callOpenAIChatCompletions) }), + ...(calls.callOpenAIResponses === undefined ? {} : { + openaiResponses: stubChatProviderPipeline('openaiResponses', calls.callOpenAIResponses), + openaiResponsesCompact: stubChatProviderPipeline('openaiResponsesCompact', calls.callOpenAIResponses), + }), + ...(calls.callAnthropicMessages === undefined ? {} : { anthropicMessages: stubChatProviderPipeline('anthropicMessages', calls.callAnthropicMessages) }), + ...(calls.callAnthropicMessagesCountTokens === undefined ? {} : { anthropicMessagesCountTokens: stubChatProviderPipeline('anthropicMessagesCountTokens', calls.callAnthropicMessagesCountTokens) }), +}); + +export const chatFixtureHttpServices = (gateway: GatewayCtx): ProviderChatServices => ({ + httpCall: () => ({ ...noopUpstreamCallOptions(), signal: gateway.abortSignal }), + recordProtocolFrames: frames => frames, +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts index 273e0304ca..99b9e0d0f6 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts @@ -28,11 +28,11 @@ export const answerClaudeCodeProbe = defineStage< Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers'>, Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers'>, Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' - | 'response.usage.billable' | 'response.http.headers'>, + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' - | 'response.usage.billable' | 'response.http.headers'>, + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' - | 'response.usage.billable' | 'response.http.headers'>, + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, ChatServices >({ name: 'answerClaudeCodeProbe', @@ -53,7 +53,7 @@ export const answerClaudeCodeProbe = defineStage< 'response.chat.anthropicMessages', 'response.chat.anthropicMessages.streamedUsage', 'response.usage.billable', - 'response.http.headers', + 'response.http.headers', 'response.http.body', ], }, execute: async (facts, next, use) => { @@ -64,7 +64,7 @@ export const answerClaudeCodeProbe = defineStage< // The probe is answered *for* this candidate, so it is the one a follow-up turn carrying // our own state must come back to — the same statement a dialled attempt makes. use.selectAffinity(candidate); - use.log.debug('answering a Claude Code probe without dialling', { upstream: facts['route.attempt'].upstreamId }); + await use.log.debug('answering a Claude Code probe without dialling', { upstream: facts['route.attempt'].upstreamId }); return move({ ...facts, 'response.chat.anthropicMessages': { @@ -79,6 +79,7 @@ export const answerClaudeCodeProbe = defineStage< quantities: {}, }], 'response.http.headers': [], + 'response.http.body': null, }); }, }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/call-upstream.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/call-upstream.ts index bf87ccafb2..7082022535 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/call-upstream.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/call-upstream.ts @@ -1,107 +1,44 @@ import type { Fields } from './facts.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; -import type { BillableEntity } from '../../pipeline/facts.ts'; -import { upstreamPerformanceContext, telemetryModelIdentity } from '../../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; +import { providerEntry } from '../../pipeline/provider-entry.ts'; +import { upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; import type { ChatServices } from '../services.ts'; import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; +import { providerChatAnswer } from '../shared/provider-answer.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { parseAnthropicBetaHeader } from '@floway-dev/protocols/anthropic-messages'; -import { providerModelOf } from '@floway-dev/provider'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; -/** - * The wire. What it hands up is the upstream's own frames, unread: the reading is taken at the - * top of the wire, above every rule that rewrites them, so nothing here has the means to - * produce a figure the client will not be shown. - */ export const callAnthropicMessagesUpstream = defineStage< Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, - Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers'>, + ProviderOperationRequest<'anthropicMessages'>, + ProviderChatResponse<'anthropicMessages'>, + Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, ChatServices >({ name: 'callAnthropicMessagesUpstream', - return: { - provides: [ - 'response.chat.anthropicMessages', - 'response.usage.billable', - 'response.http.headers', - ], + into: { + request: { needs: ['request.chat.anthropicMessages', 'route.attempt', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', 'request.provider.anthropicBeta'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - - // What the record holds by now: the payload affinity materialized for this candidate, as - // every stage between the fork and here has rewritten it — or, on a translated wire, what - // the handoff put here. A thinking signature was rewritten for the upstream that will see - // it, and one that no upstream but the issuer can read was dropped rather than sent on. const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamAnthropicMessages); - - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. Anthropic's beta flags are the exception — they have a - // typed path of their own so no header allowlist can admit them, and they are the - // client's own only when the client spoke this protocol: a turn that arrived here through - // a translation asked for nothing on this wire, so the field is read off and then dropped - // whichever protocol sent it. const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); - const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'anthropicMessages' - ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) - : []; + const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'anthropicMessages' ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) : []; headers.delete('anthropic-beta'); - - let result; - try { - result = await candidate.provider.instance.callAnthropicMessages( - providerModelOf(candidate), - body, - use.gateway.abortSignal, - { ...buildUpstreamCallOptions(candidate, use.gateway, headers), anthropicBeta }, - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.chat.anthropicMessages': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.usage.billable': [], - 'response.http.headers': [], - }); - } - - const identity = telemetryModelIdentity(candidate, result.modelKey); - // An upstream that was called and reported nothing, which is a different statement from - // reporting zero. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; - - if (!result.ok) { - const text = await result.response.text(); - use.log.warn('upstream refused', { status: result.response.status }); - let parsed: unknown; - try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } - return move({ - ...facts, - 'response.chat.anthropicMessages': { - status: result.response.status, - message: text, - ...(parsed === undefined ? {} : { body: parsed }), - }, - 'response.usage.billable': called, - 'response.http.headers': [...result.response.headers], - }); - } - - // This candidate answered, so it is the one a follow-up turn carrying our own state - // must come back to. - use.selectAffinity(candidate); + const pipeline = candidate.provider.pipelines.anthropicMessages; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no anthropicMessages pipeline`); + const back = await next(move({ ...providerEntry(facts, candidate, body, [...headers]), 'request.provider.anthropicBeta': anthropicBeta }), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = reply.answer; + if ('kind' in answer) use.selectAffinity(candidate); return move({ - ...facts, - 'response.chat.anthropicMessages': { kind: 'stream' as const, frames: result.events }, - 'response.usage.billable': called, - 'response.http.headers': [...(result.headers ?? new Headers())], + ...rest, 'response.chat.anthropicMessages': answer, + 'response.usage.billable': reply.billable, 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, }); }, }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index c21b1bc005..c11ae129be 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -21,7 +21,7 @@ export const anthropicMessagesCountTokensPipeline = (payload: AnthropicMessagesP resolveChatCandidates(narrowing(payload)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), - owns: [], + owns: ['response.http.body'], }), materializeAttempt('request.chat.anthropicMessages'), ...anthropicMessagesCountTokensWire, diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/call-upstream.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/call-upstream.ts index 8bb67caf41..5eea866b48 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/call-upstream.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/call-upstream.ts @@ -1,88 +1,41 @@ -import type { Fields, Counted } from './facts.ts'; import { bodyForAttempt } from '../../../pipeline/attempt-body.ts'; -import { buildUpstreamCallOptions } from '../../../shared/upstream-call-options.ts'; +import { providerEntry } from '../../../pipeline/provider-entry.ts'; import type { ChatServices } from '../../services.ts'; import { applyRulesToUpstreamAnthropicMessages } from '../../shared/alias-rules.ts'; +import { providerChatAnswer } from '../../shared/provider-answer.ts'; +import type { Fields } from '../facts.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { parseAnthropicBetaHeader } from '@floway-dev/protocols/anthropic-messages'; -import { providerModelOf } from '@floway-dev/provider'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; -/** - * The ending. It asks the upstream what the turn would cost and hands the answer up as a - * value — the one shape this operation ever produces, because a measurement is a body and - * never a stream. - */ export const callAnthropicMessagesCountTokensUpstream = defineStage< Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, - Counted<'response.chat.anthropicMessages'> & Fields<'response.usage.billable' | 'response.http.headers'>, + ProviderOperationRequest<'anthropicMessagesCountTokens'>, + ProviderChatResponse<'anthropicMessagesCountTokens'>, + Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, ChatServices >({ name: 'callAnthropicMessagesCountTokensUpstream', - return: { - provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers'], + into: { + request: { needs: ['request.chat.anthropicMessages', 'route.attempt', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', 'request.provider.anthropicBeta'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - - // The payload affinity materialized for this candidate, as every stage between the fork - // and here has rewritten it — built for the dial exactly as generation builds it, so what - // is measured is what generation would be charged for. const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamAnthropicMessages); - - // Anthropic's beta flags have a typed path of their own, so no header allowlist can admit - // them, and they are the client's own only when the client spoke this protocol. const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); - const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'anthropicMessages' - ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) - : []; + const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'anthropicMessages' ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) : []; headers.delete('anthropic-beta'); - - // Nothing here is billed: measuring is not generating, and an upstream that answered - // the question charged nothing for it. - const nothingBilled = { 'response.usage.billable': [] as const }; - - let response: Response; - try { - ({ response } = await candidate.provider.instance.callAnthropicMessagesCountTokens( - providerModelOf(candidate), - body, - use.gateway.abortSignal, - { ...buildUpstreamCallOptions(candidate, use.gateway, headers), anthropicBeta }, - )); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so there are no headers to carry. - return move({ - ...facts, - ...nothingBilled, - 'response.chat.anthropicMessages': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.http.headers': [], - }); - } - - const text = await response.text(); - let parsed: unknown; - try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } - - if (!response.ok) { - use.log.warn('upstream refused', { status: response.status }); - return move({ - ...facts, - ...nothingBilled, - 'response.chat.anthropicMessages': { - status: response.status, - message: text, - ...(parsed === undefined ? {} : { body: parsed }), - }, - 'response.http.headers': [...response.headers], - }); - } - + const pipeline = candidate.provider.pipelines.anthropicMessagesCountTokens; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no anthropicMessagesCountTokens pipeline`); + const back = await next(move({ ...providerEntry(facts, candidate, body, [...headers]), 'request.provider.anthropicBeta': anthropicBeta }), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = reply.answer; return move({ - ...facts, - ...nothingBilled, - 'response.chat.anthropicMessages': { kind: 'value' as const, body: parsed }, - 'response.http.headers': [...response.headers], + ...rest, 'response.chat.anthropicMessages': answer, + 'response.usage.billable': [], 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, }); }, }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts index 4c75256423..c527b0c0e1 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts @@ -24,7 +24,7 @@ export const prepareAnthropicMessagesWebSearchRequest = defineStage< Fields<'request.chat.anthropicMessages'>, Counted<'response.chat.anthropicMessages'>, Counted<'response.chat.anthropicMessages'>, - Counted<'response.chat.anthropicMessages'> & Fields<'response.usage.billable' | 'response.http.headers'>, + Counted<'response.chat.anthropicMessages'> & Fields<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, ChatServices >({ name: 'prepareAnthropicMessagesWebSearchRequest', @@ -36,7 +36,7 @@ export const prepareAnthropicMessagesWebSearchRequest = defineStage< }, response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, }, - return: { provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers'] }, + return: { provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.body'] }, execute: async (facts, next) => { if (!facts['route.attempt'].flags.includes('anthropic-messages-web-search-shim')) return await next(facts); const prepared = prepareAnthropicMessagesWebSearchShimRequest(facts['request.chat.anthropicMessages']); @@ -48,6 +48,7 @@ export const prepareAnthropicMessagesWebSearchRequest = defineStage< 'response.chat.anthropicMessages': { status: 400, message: prepared.message }, 'response.usage.billable': [], 'response.http.headers': [], + 'response.http.body': null, }); } // A request that engaged nothing comes back by identity, so the record shows no change diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts index 9e72377991..e66b5d1edc 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts @@ -6,7 +6,7 @@ import { isFailure, renderFailure, mintedAs } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { bindClientRelease, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { collectAnthropicMessagesProtocolEventsToResult, anthropicMessagesEventToSsePayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; @@ -63,7 +63,7 @@ export const emitAnthropicMessages = defineStage< 'response.chat.clientFrames': null, 'response.http.headers': forClient, 'response.chat.anthropicMessages.rendered': move(answer.body as Record), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; } @@ -84,7 +84,7 @@ export const emitAnthropicMessages = defineStage< 'response.chat.anthropicMessages.rendered': move( await collectAnthropicMessagesProtocolEventsToResult(frames) as unknown as Record, ), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; } const completed = framedClientStream( @@ -93,13 +93,14 @@ export const emitAnthropicMessages = defineStage< error => [eventFrame({ type: 'error' as const, error: toInternalDebugError(error, 'anthropicMessages') })], use.gateway.dump, ); + bindClientRelease(back, completed.release); return { ...rest, 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, 'response.chat.anthropicMessages.rendered': move(completed.rendered), 'response.chat.anthropicMessages.streamedUsage': move(withClientVerdict(back['response.chat.anthropicMessages.streamedUsage'], completed.failed)), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; }, }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts index 9f7e36d4e6..601322ac69 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts @@ -50,6 +50,7 @@ const meterAnthropicMessages = ( if (isAnthropicMessagesTerminalFrame(frame)) { sawTerminal = true; failed = frame.type === 'event' && frame.event.type === 'error'; + settle({ billable: [billedEntity(reported, identity)], failed }); } yield frame; // The turn is over, so there is nothing further to read. An upstream that holds the diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 5b242ba5b8..e2dce273da 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -22,7 +22,7 @@ export const anthropicMessagesServePipeline = (payload: AnthropicMessagesPayload resolveChatCandidates(narrowing(payload)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), - owns: [], + owns: ['response.http.body'], pendingUsage: STREAMED_USAGE, }), materializeAttempt('request.chat.anthropicMessages'), diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts index 6cff05cd0d..d4b563b6fc 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts @@ -70,7 +70,7 @@ export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => de // A declaration this gateway cannot execute, and a search backend the operator has not // configured, are both answered here: neither is a body an upstream should be asked about. return: { - provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.chat.anthropicMessages.streamedUsage'], + provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.chat.anthropicMessages.streamedUsage'], }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -86,13 +86,14 @@ export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => de const prepared = prepareAnthropicMessagesWebSearchInvocation(invocation); if (prepared.type === 'inactive') return await next(facts); if (prepared.type === 'invalid-request') { - use.log.debug('refusing a web-search declaration this gateway cannot execute'); + await use.log.debug('refusing a web-search declaration this gateway cannot execute'); return move({ ...facts, 'response.chat.anthropicMessages': refusal(prepared.message), 'response.chat.anthropicMessages.streamedUsage': null, 'response.usage.billable': [], 'response.http.headers': [], + 'response.http.body': null, }) as never; } diff --git a/packages/gateway/src/data-plane/chat/dial-wire.ts b/packages/gateway/src/data-plane/chat/dial-wire.ts index 38b1bb9047..d6fb93035e 100644 --- a/packages/gateway/src/data-plane/chat/dial-wire.ts +++ b/packages/gateway/src/data-plane/chat/dial-wire.ts @@ -51,7 +51,7 @@ export const dialChatWire = (wiring: ChatWiring) => defineStage< execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); const target = wiring.pick(candidate.model.endpoints); - use.log.debug('dialling', { upstream: facts['route.attempt'].upstreamId, wire: target }); + await use.log.debug('dialling', { upstream: facts['route.attempt'].upstreamId, wire: target }); return await next(facts, wiring.wire(target, candidate, use)); }, }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index 2b061930b8..6ecc6b5c84 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -25,7 +25,7 @@ export const geminiGenerateContentCountTokensPipeline = (payload: GeminiGenerate resolveChatCandidates(narrowing(payload)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), - owns: [], + owns: ['response.http.body'], }), materializeAttempt('request.chat.geminiGenerateContent'), stripUnsupportedPartFieldsFromGeminiGenerateContent, diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts index dd45f89c13..e9c2aa895a 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts @@ -21,7 +21,7 @@ export const measureGeminiGenerateContentAsAnthropicMessages = defineStage< Fields<'request.chat.anthropicMessages'>, Counted<'response.chat.anthropicMessages'>, Counted<'response.chat.geminiGenerateContent'>, - Counted<'response.chat.geminiGenerateContent'> & Fields<'response.usage.billable' | 'response.http.headers'>, + Counted<'response.chat.geminiGenerateContent'> & Fields<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, ChatServices >({ name: 'measureGeminiGenerateContentAsAnthropicMessages', @@ -37,7 +37,7 @@ export const measureGeminiGenerateContentAsAnthropicMessages = defineStage< provides: ['response.chat.geminiGenerateContent'], }, }, - return: { provides: ['response.chat.geminiGenerateContent', 'response.usage.billable', 'response.http.headers'] }, + return: { provides: ['response.chat.geminiGenerateContent', 'response.usage.billable', 'response.http.headers', 'response.http.body'] }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); const { 'request.chat.geminiGenerateContent': asked, ...down } = facts; @@ -62,6 +62,7 @@ export const measureGeminiGenerateContentAsAnthropicMessages = defineStage< // never opens a stream, which is why this chain carries no such key at all. 'response.usage.billable': [], 'response.http.headers': [], + 'response.http.body': null, }); } diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts index 0d9ffed020..3efe0bfb2d 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts @@ -6,7 +6,7 @@ import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { bindClientRelease, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectGeminiGenerateContentProtocolEventsToResult, type GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; @@ -66,7 +66,7 @@ export const emitGeminiGenerateContent = defineStage< 'response.chat.clientFrames': null, 'response.http.headers': forClient, 'response.chat.geminiGenerateContent.rendered': move(answer.body as Record), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; } @@ -87,7 +87,7 @@ export const emitGeminiGenerateContent = defineStage< 'response.chat.geminiGenerateContent.rendered': move( await collectGeminiGenerateContentProtocolEventsToResult(frames) as unknown as Record, ), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; } const completed = framedClientStream( @@ -96,13 +96,14 @@ export const emitGeminiGenerateContent = defineStage< error => [eventFrame(geminiStreamError(error))], use.gateway.dump, ); + bindClientRelease(back, completed.release); return { ...rest, 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, 'response.chat.geminiGenerateContent.rendered': move(completed.rendered), 'response.chat.geminiGenerateContent.streamedUsage': move(withClientVerdict(back['response.chat.geminiGenerateContent.streamedUsage'], completed.failed)), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; }, }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index f0d12b447b..e80708838a 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -24,7 +24,7 @@ export const geminiGenerateContentServePipeline = (payload: GeminiGenerateConten resolveChatCandidates(narrowing(payload)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), - owns: [], + owns: ['response.http.body'], pendingUsage: STREAMED_USAGE, }), materializeAttempt('request.chat.geminiGenerateContent'), diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index 644072318f..ac70a782fc 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -42,6 +42,7 @@ export interface Handoff => ({ 'response.usage.billable': [], 'response.http.headers': [], + 'response.http.body': null, [`${response}.streamedUsage`]: null, [response]: { status: 400, message }, }); @@ -72,7 +73,7 @@ export const handOff = = ( attempt: { firstOutputTokenAt: number | null }, ) => { readonly frames: AsyncIterable>; readonly outcome: Deferred }; -/** Which identity the figure is attributed to, read off the entity the dial recorded when it - * called an upstream. That entity is already the statement "this attempt bills to this - * identity, and nothing has been reported for it yet" — so the meter fills in what it turned - * out to be billable for rather than deriving a second identity of its own. - * - * A wire that hands up a stream has been dialled, so there is exactly one. Anything else is - * this gateway contradicting itself, and it says so rather than billing to a guess. */ +/** Credential retries precede the final called entity. Only the final reply has frames + * to meter; earlier endpoint replies retain their own empty-quantity observations. */ const attributedTo = (billable: readonly BillableEntity[], wire: string): TelemetryModelIdentity => { - if (billable.length !== 1) { - throw new Error(`${wire} handed up a stream billable to ${billable.length} entities, and a metered stream bills to the one that was dialled`); + if (billable.length === 0) { + throw new Error(`${wire} handed up a stream billable to ${billable.length} entities, and a metered stream requires a called model identity`); } - return billable[0].identity; + return billable[billable.length - 1].identity; }; /** @@ -98,10 +93,22 @@ export const meterChatWire = (spec: { attributedTo(back['response.usage.billable'], spec.wire), use.gateway.attempt.timing, ); + const iterator = metered.frames[Symbol.asyncIterator](); + // Downstream transforms may stop after a terminal or cancellation. The body owner keeps + // this single reading alive so release can finish the upstream's billable usage. + const ownsBody = 'response.http.body' in back && back['response.http.body'] !== null; + const frames = { ...streamReferenceOf(answer.frames), [Symbol.asyncIterator]: () => ownsBody ? { next: () => iterator.next() } : iterator }; + if ('response.http.body' in back && back['response.http.body'] !== null) { + setRelease(back['response.http.body'] as ReadableStream & Owned, async () => { + let step = await iterator.next(); + while (!step.done) step = await iterator.next(); + }); + } + const prior = back['response.usage.billable'].slice(0, -1); return move({ ...back, - [spec.answer]: { kind: 'stream' as const, frames: { ...streamReferenceOf(answer.frames), [Symbol.asyncIterator]: () => metered.frames[Symbol.asyncIterator]() } }, - [spec.streamedUsage]: metered.outcome, + [spec.answer]: { kind: 'stream' as const, frames }, + [spec.streamedUsage]: prior.length === 0 ? metered.outcome : defer(metered.outcome.then(outcome => ({ ...outcome, billable: [...prior, ...outcome.billable] }))), }) as never; }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/call-upstream.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/call-upstream.ts index bb5df472c7..c0f3b77b14 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/call-upstream.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/call-upstream.ts @@ -1,100 +1,40 @@ import type { Fields } from './facts.ts'; import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; -import type { BillableEntity } from '../../pipeline/facts.ts'; -import { upstreamPerformanceContext, telemetryModelIdentity } from '../../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; +import { providerEntry } from '../../pipeline/provider-entry.ts'; +import { upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; import type { ChatServices } from '../services.ts'; import { applyRulesToUpstreamOpenAIChatCompletions } from '../shared/alias-rules.ts'; +import { providerChatAnswer } from '../shared/provider-answer.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import { providerModelOf } from '@floway-dev/provider'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; -/** - * The wire. It dials OpenAI Chat Completions and provides the answer at whichever family's response - * key the chain above it reads — which is what makes it interchangeable with a translated - * chain: both hand up `response.chat.openaiChatCompletions`, and the stage above cannot tell which - * ran. - * - * What it hands up is the upstream's own frames, unread. The reading is taken at the top of the - * wire, above every rule that rewrites them, so nothing here has the means to produce a figure - * the client will not be shown. - */ export const callOpenAIChatCompletionsUpstream = defineStage< Fields<'request.chat.openaiChatCompletions' | 'route.attempt' | 'ingress.http.headers'>, - Fields<'response.chat.openaiChatCompletions' | 'response.usage.billable' | 'response.http.headers'>, + ProviderOperationRequest<'openaiChatCompletions'>, + ProviderChatResponse<'openaiChatCompletions'>, + Fields<'response.chat.openaiChatCompletions' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, ChatServices >({ name: 'callOpenAIChatCompletionsUpstream', - return: { - provides: [ - 'response.chat.openaiChatCompletions', - 'response.usage.billable', - 'response.http.headers', - ], + into: { + request: { needs: ['request.chat.openaiChatCompletions', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.openaiChatCompletions', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - - // What the record holds by now: the payload affinity materialized for this candidate, - // as every stage between the fork and here has rewritten it — or, on a translated wire, - // what the handoff put here. const body = bodyForAttempt(facts['request.chat.openaiChatCompletions'], candidate, applyRulesToUpstreamOpenAIChatCompletions); - - let result; - try { - result = await candidate.provider.instance.callOpenAIChatCompletions( - providerModelOf(candidate), - body, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.chat.openaiChatCompletions': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.usage.billable': [], - 'response.http.headers': [], - }); - } - - const identity = telemetryModelIdentity(candidate, result.modelKey); - // An upstream that was called and reported nothing, which is a different statement from - // reporting zero. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; - - if (!result.ok) { - const text = await result.response.text(); - use.log.warn('upstream refused', { status: result.response.status }); - let parsed: unknown; - try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } - return move({ - ...facts, - 'response.chat.openaiChatCompletions': { - status: result.response.status, - message: text, - ...(parsed === undefined ? {} : { body: parsed }), - }, - 'response.usage.billable': called, - 'response.http.headers': [...result.response.headers], - }); - } - - // This candidate answered, so it is the one a follow-up turn carrying our own state - // must come back to. - use.selectAffinity(candidate); + const pipeline = candidate.provider.pipelines.openaiChatCompletions; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no openaiChatCompletions pipeline`); + const back = await next(providerEntry(facts, candidate, body), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = reply.answer; + if ('kind' in answer) use.selectAffinity(candidate); return move({ - ...facts, - 'response.chat.openaiChatCompletions': { kind: 'stream' as const, frames: result.events }, - 'response.usage.billable': called, - 'response.http.headers': [...(result.headers ?? new Headers())], + ...rest, 'response.chat.openaiChatCompletions': answer, + 'response.usage.billable': reply.billable, 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, }); }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts index 6665b775c2..1511948f88 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts @@ -5,7 +5,7 @@ import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/fa import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { bindClientRelease, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { eventFrame, isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectOpenAIChatCompletionsProtocolEventsToResult, openaiChatCompletionsProtocolFrameToSSEFrame, type OpenAIChatCompletionsStreamEvent, type ClientOpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; @@ -65,7 +65,7 @@ export const emitOpenAIChatCompletions = defineStage< 'response.chat.clientFrames': null, 'response.http.headers': forClient, 'response.chat.openaiChatCompletions.rendered': move(answer.body as Record), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; } @@ -86,7 +86,7 @@ export const emitOpenAIChatCompletions = defineStage< 'response.chat.openaiChatCompletions.rendered': move( await collectOpenAIChatCompletionsProtocolEventsToResult(frames) as unknown as Record, ), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; } const completed = framedClientStream>( @@ -95,13 +95,14 @@ export const emitOpenAIChatCompletions = defineStage< error => [eventFrame({ error: toInternalDebugError(error, 'openaiChatCompletions') })], use.gateway.dump, ); + bindClientRelease(back, completed.release); return { ...rest, 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, 'response.chat.openaiChatCompletions.rendered': move(completed.rendered), 'response.chat.openaiChatCompletions.streamedUsage': move(withClientVerdict(back['response.chat.openaiChatCompletions.streamedUsage'], completed.failed)), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts index 9d515c8a8c..f6a04f2bb0 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts @@ -47,6 +47,7 @@ const meterOpenAIChatCompletions = ( if (isTerminal(frame)) { completed = true; failed = frame.type === 'event' && 'error' in frame.event; + settle({ billable: [billedEntity(reported, identity)], failed }); } yield frame; // The terminator is written out before the read stops, because it is what the client diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index ff1fcf0304..90e1258ea7 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -20,7 +20,7 @@ export const openaiChatCompletionsServePipeline = (payload: OpenAIChatCompletion resolveChatCandidates(narrowing(payload)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.openaiChatCompletions'?: unknown })['response.chat.openaiChatCompletions']), - owns: [], + owns: ['response.http.body'], pendingUsage: STREAMED_USAGE, }), materializeAttempt('request.chat.openaiChatCompletions'), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts b/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts index 6dea23cf67..50d48958e8 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts @@ -36,6 +36,7 @@ export const answerOpenAIResponsesWebSocketWarmup = defineStage, - Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'>, + ProviderOperationRequest<'openaiResponses'>, + ProviderChatResponse<'openaiResponses'>, + Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, ChatServices >({ name: 'callOpenAIResponsesUpstream', - return: { - provides: [ - 'response.chat.openaiResponses', - 'response.usage.billable', - 'response.http.headers', - ], + into: { + request: { needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.openaiResponses', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - - // What the record holds by now: the payload affinity materialized for this candidate, as - // every stage between the fork and here has rewritten it — or, on a translated wire, what - // the handoff put here. Client-carried state — an encrypted reasoning blob, a compaction - // the upstream issued — was rewritten for the upstream that will see it, which is the - // whole reason a turn can be pinned at all. - // - // The key holds what a client may send, whose `input` is a string or a list; this chain - // runs on the canonical form the entry normalized it to, which is the one a wire takes. - const asked = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; - const body = bodyForAttempt(asked, candidate, applyRulesToUpstreamOpenAIResponses); - - let result; - try { - result = await candidate.provider.instance.callOpenAIResponses( - providerModelOf(candidate), - body, - 'generate', - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.chat.openaiResponses': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.usage.billable': [], - 'response.http.headers': [], - }); - } - - const identity = telemetryModelIdentity(candidate, result.modelKey); - // An upstream that was called and reported nothing, which is a different statement from - // reporting zero. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; - - if (!result.ok) { - const text = await result.response.text(); - use.log.warn('upstream refused', { status: result.response.status }); - let parsed: unknown; - try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } - return move({ - ...facts, - 'response.chat.openaiResponses': { - status: result.response.status, - message: text, - ...(parsed === undefined ? {} : { body: parsed }), - }, - 'response.usage.billable': called, - 'response.http.headers': [...result.response.headers], - }); - } - - // This candidate answered, so it is the one a follow-up turn carrying our own state must - // come back to. - use.selectAffinity(candidate); - - if (result.action === 'compact') { - return move({ - ...facts, - 'response.chat.openaiResponses': { kind: 'stream' as const, frames: syntheticEventsFromCompaction(result.result) }, - 'response.usage.billable': called, - 'response.http.headers': [], - }); - } - + const body = bodyForAttempt(facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload, candidate, applyRulesToUpstreamOpenAIResponses); + const pipeline = candidate.provider.pipelines.openaiResponses; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no openaiResponses pipeline`); + const back = await next(providerEntry(facts, candidate, body), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = 'kind' in reply.answer && reply.answer.kind === 'value' + ? { kind: 'stream' as const, frames: use.recordProtocolFrames(syntheticEventsFromCompaction(reply.answer.body as Parameters[0])) } + : reply.answer; + if ('kind' in answer) use.selectAffinity(candidate); return move({ - ...facts, - 'response.chat.openaiResponses': { kind: 'stream' as const, frames: result.events }, - 'response.usage.billable': called, - 'response.http.headers': [...(result.headers ?? new Headers())], + ...rest, 'response.chat.openaiResponses': answer, + 'response.usage.billable': reply.billable, 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, }); }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact-decrypt.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact-decrypt.ts index 47d2478b8e..f51441acd9 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact-decrypt.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact-decrypt.ts @@ -3,11 +3,12 @@ import { internalErrorEnvelope } from './errors.ts'; import { syntheticEventsFromCompaction, syntheticEventsFromResult } from './items/output.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import { providerCalls } from '../../pipeline/provider-usage.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import type { ChatFacts, ChatAnswer } from '../facts.ts'; import type { ChatServices } from '../services.ts'; import type { ChatWire } from '../wire.ts'; -import { defer, defineStage, move, type Deferred } from '@floway-dev/pipeline'; +import { defer, defineStage, getFailureFacts, move, type Deferred } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { collectOpenAIResponsesProtocolEventsToResult, @@ -18,7 +19,7 @@ import { type OpenAIResponsesStreamEvent, } from '@floway-dev/protocols/openai-responses'; -type Request = Pick; +type Request = Pick; type Answer = Pick & Record; export const decryptNativeCompaction = (spec: { @@ -30,11 +31,11 @@ export const decryptNativeCompaction = (spec: { }) => defineStage({ name: 'decryptNativeCompaction', into: { - request: { needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], consumes: [], provides: ['request.chat.openaiResponses'] }, + request: { needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers', 'ingress.chat.sourceProtocol', 'serve.usage.prior'], consumes: [], provides: ['request.chat.openaiResponses'] }, response: { needs: ['response.chat.openaiResponses', spec.streamedUsage, 'response.usage.billable', 'response.http.headers'], - consumes: [], - provides: ['response.chat.openaiResponses', spec.streamedUsage, 'response.usage.billable'], + consumes: ['response.http.body'], + provides: ['response.chat.openaiResponses', spec.streamedUsage, 'response.usage.billable', 'response.http.body'], }, }, execute: async (facts, next, use) => { @@ -86,7 +87,14 @@ export const decryptNativeCompaction = (spec: { ], store: false, }; - const replay = await next({ ...facts, 'request.chat.openaiResponses': move(replayPayload) }, spec.replay as never); + let replay: Answer; + try { + replay = await next({ ...facts, 'request.chat.openaiResponses': move(replayPayload), 'serve.usage.prior': move([...facts['serve.usage.prior'], ...billed]) }, spec.replay as never); + } catch (error) { + const atFailure = getFailureFacts(error); + if (atFailure !== undefined) billed.push(...providerCalls(use.resolveAttempt(facts['route.attempt']), atFailure)); + throw error; + } const replayAnswer = replay['response.chat.openaiResponses']; if (isFailure(replayAnswer)) { billed.push(...replay['response.usage.billable']); @@ -102,6 +110,7 @@ export const decryptNativeCompaction = (spec: { const result = { ...native, ...(!spec.compactEndpoint ? { object: 'response' as const } : {}), output, ...(usage === undefined ? {} : { usage }) }; return returned(initial, { kind: 'stream', frames: spec.compactEndpoint ? syntheticEventsFromCompaction(result) : syntheticEventsFromResult(result) }, false); } catch (error) { + use.gateway.dump?.failed(error); const envelope = internalErrorEnvelope(error); return returned(initial, { status: 500, message: envelope.error.message, envelope }, true); } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index af2378f9ab..f22ef1cdb6 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -32,7 +32,7 @@ export const openaiResponsesCompactPipeline = (payload: CanonicalOpenAIResponses resolveChatCandidates(openaiResponsesNarrowing(() => prepared)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), - owns: [], + owns: ['response.http.body'], pendingUsage: OPENAI_RESPONSES_STREAMED_USAGE, }), materializeAttempt('request.chat.openaiResponses'), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts index a7e5619c74..bfa554ee45 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts @@ -1,108 +1,45 @@ -import type { Fields } from '../facts.ts'; -import type { Compacted } from './facts.ts'; import { bodyForAttempt } from '../../../pipeline/attempt-body.ts'; -import { upstreamPerformanceContext, telemetryModelIdentity } from '../../../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../../../shared/upstream-call-options.ts'; +import { providerEntry } from '../../../pipeline/provider-entry.ts'; +import { upstreamPerformanceContext } from '../../../shared/telemetry/attribution.ts'; import type { ChatServices } from '../../services.ts'; import { applyRulesToUpstreamOpenAIResponses } from '../../shared/alias-rules.ts'; -import { OPENAI_RESPONSES_STREAMED_USAGE } from '../facts.ts'; +import { providerChatAnswer } from '../../shared/provider-answer.ts'; +import type { Fields } from '../facts.ts'; import { syntheticEventsFromCompaction } from '../items/output.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; -/** - * The compaction wire. It asks the upstream's own `/responses` endpoint to compact, and hands - * the envelope up as the events it expands into — the same currency a generate wire hands up, - * which is what lets the stateful half above read either without knowing which ran. - */ export const callOpenAIResponsesCompactUpstream = defineStage< Fields<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, - Compacted<'response.chat.openaiResponses'> & Fields<'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, + ProviderOperationRequest<'openaiResponsesCompact'>, + ProviderChatResponse<'openaiResponsesCompact'>, + Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body' | 'response.chat.openaiResponses.streamedUsage'>, ChatServices >({ name: 'callOpenAIResponsesCompactUpstream', - return: { - provides: [ - 'response.chat.openaiResponses', - OPENAI_RESPONSES_STREAMED_USAGE, - 'response.usage.billable', - 'response.http.headers', - ], + into: { + request: { needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.openaiResponses', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body', 'response.chat.openaiResponses.streamedUsage'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); - - const asked = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; - // Neither field belongs on this endpoint: `store` is a gateway-only snapshot hint the - // compaction endpoint rejects, and `stream` describes how an answer would be delivered - // where there is one body and no stream. - const { stream: _stream, store: _store, ...body } = bodyForAttempt(asked, candidate, applyRulesToUpstreamOpenAIResponses); - - let result; - try { - result = await candidate.provider.instance.callOpenAIResponses( - providerModelOf(candidate), - body, - 'compact', - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.chat.openaiResponses': { status: 502, message: error instanceof Error ? error.message : String(error) }, - [OPENAI_RESPONSES_STREAMED_USAGE]: null, - 'response.usage.billable': [], - 'response.http.headers': [], - }); - } - - const identity = telemetryModelIdentity(candidate, result.modelKey); - if (!result.ok) { - const text = await result.response.text(); - use.log.warn('upstream refused', { status: result.response.status }); - let parsed: unknown; - try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } - return move({ - ...facts, - 'response.chat.openaiResponses': { - status: result.response.status, - message: text, - ...(parsed === undefined ? {} : { body: parsed }), - }, - [OPENAI_RESPONSES_STREAMED_USAGE]: null, - // An upstream that was called and reported nothing, which is a different statement - // from reporting zero. - 'response.usage.billable': [{ identity, quantities: {} }], - 'response.http.headers': [...result.response.headers], - }); - } - if (result.action !== 'compact') { - throw new Error(`callOpenAIResponsesCompactUpstream: ${facts['route.attempt'].upstreamId} answered a compaction dial with a ${result.action} turn`); - } - - // This candidate answered, so it is the one a follow-up turn carrying our own state must - // come back to. - use.selectAffinity(candidate); - - const billable = [{ identity, quantities: {} }]; + const { stream: _stream, store: _store, ...body } = bodyForAttempt(facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload, candidate, applyRulesToUpstreamOpenAIResponses); + const pipeline = candidate.provider.pipelines.openaiResponsesCompact; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no openaiResponsesCompact pipeline`); + const back = await next(providerEntry(facts, candidate, body), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = 'kind' in reply.answer && reply.answer.kind === 'value' + ? { kind: 'stream' as const, frames: use.recordProtocolFrames(syntheticEventsFromCompaction(reply.answer.body as Parameters[0])) } + : reply.answer; + if ('kind' in answer) use.selectAffinity(candidate); return move({ - ...facts, - 'response.chat.openaiResponses': { kind: 'stream' as const, frames: syntheticEventsFromCompaction(result.result) }, - // The outer meter reads the synthetic frames after this wire's usage normalizers. - [OPENAI_RESPONSES_STREAMED_USAGE]: null, - 'response.usage.billable': billable, - 'response.http.headers': [], + ...rest, 'response.chat.openaiResponses': answer, + 'response.usage.billable': reply.billable, 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, + 'response.chat.openaiResponses.streamedUsage': null, }); }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts index 27b18e5e1d..01c24b417d 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts @@ -48,7 +48,7 @@ export const dialOpenAIResponsesCompaction = (wires: { native: ChatWire; simulat execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); const compacts = !simulatesCompaction(candidate, facts['route.attempt']); - use.log.debug('compacting', { upstream: facts['route.attempt'].upstreamId, wire: compacts ? 'compact' : 'simulated' }); + await use.log.debug('compacting', { upstream: facts['route.attempt'].upstreamId, wire: compacts ? 'compact' : 'simulated' }); return await next(facts, compacts ? wires.native : wires.simulated); }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts index 0aa2f1310f..7ea2fc7716 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts @@ -5,7 +5,7 @@ import { recordStream } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; -import { framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { bindClientRelease, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectOpenAIResponsesProtocolEventsToResult, openaiResponsesProtocolFrameToSSEFrame, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent, type ClientOpenAIResponsesStreamEvent, type ClientResponseResource } from '@floway-dev/protocols/openai-responses'; @@ -72,7 +72,7 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra 'response.chat.clientFrames': null, 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move(answer.body as Record), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; } @@ -107,7 +107,7 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra 'response.chat.openaiResponses.rendered': move( await collectOpenAIResponsesProtocolEventsToResult(frames) as unknown as Record, ), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; } catch (error) { // Nothing has gone out yet, so the fault is still a status. A turn the gateway could @@ -127,12 +127,13 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra if (framing === 'events') { const clientStream = completeEventStream(egress, error => { use.gateway.dump?.failed(error); }); const frames = recordStream(clientStream.frames, use.gateway.dump); + bindClientRelease(back, clientStream.release); return { ...rest, 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move(frames), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], clientStream.failed)), }; } @@ -147,12 +148,13 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra error => [eventFrame(streamErrorEvent(error)), ...(announced === undefined ? [] : [eventFrame(streamFailedEvent(announced, error))])], use.gateway.dump, ); + bindClientRelease(back, completed.release); return { ...rest, 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move(completed.rendered), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], completed.failed)), }; }, @@ -192,22 +194,31 @@ const streamFailedEvent = (announced: ClientResponseResource, error: unknown): C const completeEventStream = ( frames: AsyncIterable>, reportFailure: (error: unknown) => void, -): { frames: AsyncIterable>; failed: Promise } => { +): { frames: AsyncIterable>; failed: Promise; release: () => Promise } => { let settle!: (failed: boolean) => void; const failed = new Promise(resolve => { settle = resolve; }); + let started = false; const stream = (async function* () { + started = true; let failed = false; + let completed = false; try { for await (const frame of frames) if (frame.type !== 'done') yield frame; + completed = true; } catch (error) { failed = true; reportFailure(error); throw error; } finally { - settle(failed); + settle(failed || !completed); } })(); - return { frames: { [Symbol.asyncIterator]: () => stream }, failed }; + return { + frames: { [Symbol.asyncIterator]: () => stream }, failed, release: async () => { + if (!started) settle(true); + await stream.return(); + }, + }; }; const completedSseFrames = (frames: AsyncIterable>): AsyncIterable> => ({ diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts index 4c3bf83a50..817686f745 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts @@ -24,10 +24,11 @@ import { buildErrorFromRefusal, consumeTurnStreaming, createMergeState, material import { type ActiveHostedTool, type MergeState, type HostedToolDispatcher, type HostedToolRewrite, type HostedToolLoopState, type HostedToolRegistration, type HostedToolPrepareResult, type TurnSummary } from './hosted-tools/types.ts'; import type { BillableEntity, Failure } from '../../pipeline/facts.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import { providerCalls } from '../../pipeline/provider-usage.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; import type { ChatAnswer, ChatFacts } from '../facts.ts'; import type { ChatServices } from '../services.ts'; -import { defer, defineStage, move, type Deferred } from '@floway-dev/pipeline'; +import { defer, defineStage, getFailureFacts, move, setRelease, type Owned, type Deferred } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, @@ -154,8 +155,8 @@ export const hostedTools = ( registrations: readonly HostedToolRegistration[], wiring: HostedToolWiring, ) => defineStage< - R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, - R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>, + R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>, Answered, Answered, Answered, @@ -164,20 +165,20 @@ export const hostedTools = ( name: 'hostedTools', through: { request: { - needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers'], + needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers', 'serve.usage.prior'], consumes: [], provides: ['request.chat.openaiResponses'], }, response: { needs: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers'], - consumes: [], - provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable'], + consumes: ['response.http.body'], + provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.body'], }, }, // A tool declaration this gateway cannot accept is answered here rather than dialled: the // upstream would have been asked for a tool it does not implement, on a body the dispatcher wrote. return: { - provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers'], + provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers', 'response.http.body'], }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -193,13 +194,14 @@ export const hostedTools = ( const prepared = await prepareHostedTools(registrations, invocation, use.gateway); if ('refused' in prepared) { - use.log.debug('refusing a hosted-tool declaration this gateway cannot accept'); + await use.log.debug('refusing a hosted-tool declaration this gateway cannot accept'); return move({ ...facts, 'response.chat.openaiResponses': prepared.refused, [wiring.streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], + 'response.http.body': null, }) as never; } const { active } = prepared; @@ -241,28 +243,40 @@ export const hostedTools = ( // spliced stream ends — which is the moment the run is answerable for what it billed. const usage = defer(new Promise(resolve => { settle = resolve; })); + let started = false; + let drainActive: () => Promise = async () => {}; + let closing: Promise | undefined; + const close = (): Promise => closing ??= (async () => { + await drainActive(); + if (!started) { + const outcome = first.usage === null ? { billable: back['response.usage.billable'], failed: false } : await first.usage; + settle({ ...outcome, failed: true }); + } + await iterator.return(); + })(); + const observeBody = (turn: Answered): void => { + if (!('response.http.body' in turn) || turn['response.http.body'] === null) return; + const body = turn['response.http.body'] as ReadableStream & Owned; + const drain = setRelease(body, async () => { + try { await drain(); } finally { await close(); } + }); + drainActive = drain; + }; + const iterator = (async function* () { + started = true; + yield* spliceTurns({ + next, facts, invocation, merge, loopState, demoteForcedHostedToolChoiceAfterFirstTurn, + first, dispatchers, store: use.gateway.store, canonicalInput, active, billed, settle, + streamedUsage: wiring.streamedUsage, candidate, + reportFailure: error => { use.gateway.dump?.failed(error); }, observeBody, + }); + })(); + observeBody(back); return { ...back, 'response.chat.openaiResponses': move({ kind: 'stream' as const, - frames: { - [Symbol.asyncIterator]: () => spliceTurns({ - next, - facts, - invocation, - merge, - loopState, - demoteForcedHostedToolChoiceAfterFirstTurn, - first, - dispatchers, - store: use.gateway.store, - canonicalInput, - active, - billed, - settle, - streamedUsage: wiring.streamedUsage, - }), - }, + frames: { [Symbol.asyncIterator]: () => ({ next: () => iterator.next() }) }, }), [wiring.streamedUsage]: move(usage), // What was known when this stage handed up, which is what the first call reported. The @@ -274,8 +288,8 @@ export const hostedTools = ( /** One descent, asked with the payload the dispatcher wrote. */ const descend = async ( - next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>) => Promise, - facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>) => Promise, + facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>, payload: CanonicalOpenAIResponsesPayload, ): Promise => await next(move({ ...facts, 'request.chat.openaiResponses': move(payload) })); @@ -313,8 +327,8 @@ async function* consumeBilledTurn( } async function* spliceTurns(args: { - next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>) => Promise; - facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>; + next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>) => Promise; + facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>; invocation: OpenAIResponsesInvocation; merge: MergeState; loopState: HostedToolLoopState; @@ -327,6 +341,9 @@ async function* spliceTurns(args: { billed: BillableEntity[]; settle: (outcome: StreamOutcome) => void; streamedUsage: string; + candidate: ModelCandidate; + reportFailure: (error: unknown) => void; + observeBody: (back: Answered) => void; }): AsyncGenerator> { const { invocation, merge, loopState, demoteForcedHostedToolChoiceAfterFirstTurn, dispatchers, store, active, billed, settle } = args; const baseInput = args.canonicalInput; @@ -386,7 +403,15 @@ async function* spliceTurns(args: { }; loopState.iterationCount += 1; - const back = await descend(args.next, args.facts, invocation.payload); + let back: Answered; + try { + back = await descend(args.next, { ...args.facts, 'serve.usage.prior': move([...args.facts['serve.usage.prior'], ...billed]) }, invocation.payload); + } catch (error) { + const atFailure = getFailureFacts(error); + if (atFailure !== undefined) billed.push(...providerCalls(args.candidate, atFailure)); + throw error; + } + args.observeBody(back); const nextTurn = turnOf(back, args.streamedUsage); if (nextTurn === null) { failed = true; @@ -399,6 +424,7 @@ async function* spliceTurns(args: { } } catch (error) { failed = true; + args.reportFailure(error); if (merge.lastSeenModel === null) throw error; yield synthesizeTerminalEnvelope(merge, { kind: 'failed', diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts index 5a6810a795..3f2d98399f 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts @@ -26,7 +26,8 @@ export const prepareHostedImageGeneration = defineStage< use.gateway.dump?.failed(error); return move({ ...facts, 'response.openaiImages.canonical': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.openaiImages.streamedUsage': null, 'response.http.status': 502, 'response.http.headers': [], 'response.usage.billable': [], + 'response.openaiImages.streamedUsage': null, 'response.http.status': 502, 'response.http.headers': [], + 'response.http.body': null, 'response.usage.billable': [], }); } return await next({ diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts index d511868cda..f81a1970a7 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts @@ -36,15 +36,15 @@ export const parseRetryAfterMs = (headers: Headers): number | null => { }; export const retryRateLimitedImages = defineStage< - Fields<'request.openaiImages.canonical' | 'route.attempt'>, - Fields<'request.openaiImages.canonical' | 'route.attempt'>, + Fields<'request.openaiImages.canonical' | 'route.attempt' | 'serve.usage.prior'>, + Fields<'request.openaiImages.canonical' | 'route.attempt' | 'serve.usage.prior'>, Fields<'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable' | 'response.openaiImages.streamedUsage'>, Fields<'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable' | 'response.openaiImages.streamedUsage'>, GatewayServices >({ name: 'retryRateLimitedImages', through: { - request: { needs: ['request.openaiImages.canonical', 'route.attempt'], consumes: [], provides: [] }, + request: { needs: ['request.openaiImages.canonical', 'route.attempt', 'serve.usage.prior'], consumes: [], provides: ['serve.usage.prior'] }, response: { needs: ['response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable', 'response.openaiImages.streamedUsage'], consumes: ['response.http.body'], @@ -54,13 +54,14 @@ export const retryRateLimitedImages = defineStage< execute: async (facts, next, use) => { const prior: BillableEntity[] = []; for (let retry = 0; ; retry++) { - const back = await next(facts); + const back = await next({ ...facts, 'serve.usage.prior': move([...facts['serve.usage.prior'], ...prior]) }); if (back['response.http.status'] !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) { - if (prior.length === 0) return back; + if (prior.length === 0) return { ...back, 'serve.usage.prior': facts['serve.usage.prior'] }; const pending = back['response.openaiImages.streamedUsage'] as Deferred | null; const calls = move([...prior]); return { ...back, + 'serve.usage.prior': facts['serve.usage.prior'], 'response.usage.billable': move([...calls, ...back['response.usage.billable']]), 'response.openaiImages.streamedUsage': pending === null ? null : move(defer(pending.then(outcome => ({ ...outcome, billable: move([...calls, ...outcome.billable]) })))), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts index 036ba28146..b89a583a64 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts @@ -1,27 +1,60 @@ import type { Fields, WebSearchRequest } from './facts.ts'; import type { SearchServices, WebSearchRuntime } from './services.ts'; +import { providerEntry } from '../../../../pipeline/provider-entry.ts'; +import { providerCalls } from '../../../../pipeline/provider-usage.ts'; +import { spentBody } from '../../../../pipeline/upstream-body.ts'; +import { upstreamPerformanceContext } from '../../../../shared/telemetry/attribution.ts'; +import { executeAlphaSearch } from '../../../../tools/web-search/alpha-search/execution.ts'; +import type { AlphaSearchDispatcher } from '../../../../tools/web-search/alpha-search/upstream.ts'; import { assertLocalWebSearchSupport, executeOperationToIr, parseWebSearchOperations, runBackendSearchMulti, schemaErrorIr, startBatchFetch, UnsupportedLocalWebSearchFeatureError, unsupportedLocalWebSearchFeatureIr, type WebSearchCallIR, type WebSearchOperation } from '../../../../tools/web-search/operations.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move } from '@floway-dev/pipeline'; import type { OpenAIResponsesWebSearchAction } from '@floway-dev/protocols/openai-responses'; +import type { ProviderOperationRequest, ProviderResponse } from '@floway-dev/provider'; const ITERATION_CAP = 30; export const runWebSearchCall = defineStage< Fields<'request.webSearch.canonical'>, - Fields<'response.webSearch.ir'> & { 'response.usage.billable': readonly never[] }, + ProviderOperationRequest<'alphaSearch'>, + ProviderResponse, + Fields<'response.webSearch.ir' | 'response.usage.billable'>, + Fields<'response.webSearch.ir' | 'response.usage.billable'>, SearchServices >({ name: 'runWebSearchCall', + into: { + request: { needs: ['request.webSearch.canonical'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.modelKey', 'response.provider.previousCalls'], consumes: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.modelKey', 'response.provider.previousCalls'], provides: ['response.webSearch.ir', 'response.usage.billable'] }, + }, return: { provides: ['response.webSearch.ir', 'response.usage.billable'] }, - execute: async (facts, use) => move({ - ...facts, - 'response.webSearch.ir': await execute(facts['request.webSearch.canonical'], use.webSearch), - // Search billing belongs to the backend operations; no model entity is called here. - 'response.usage.billable': [], - }), + execute: async (facts, next, use) => { + let billable: Fields<'response.usage.billable'>['response.usage.billable'] = []; + const dispatch: AlphaSearchDispatcher = async (body, _signal, headers) => { + const alpha = use.webSearch.alpha; + if (alpha === undefined) throw new Error('Alpha Search was dispatched without a selected provider'); + const candidate = await alpha.candidate; + const pipeline = candidate.provider.pipelines.alphaSearch; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no Alpha Search pipeline`); + const selector = use.rememberCandidates([candidate])[0]; + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + const { model: _callerModel, ...payload } = body; + const back = await next(providerEntry({ ...facts, 'route.attempt': selector, 'ingress.http.headers': [...headers] }, candidate, payload), pipeline); + billable = providerCalls(candidate, back as unknown as Record); + const exchange = back['response.http.exchange']; + if (exchange.type === 'transportFailure') throw exchange.error; + const response = exchangeResponse(exchange); + // Formatting reads the finite body before the child returns; release then has no bytes left. + const bytes = await response.arrayBuffer(); + spentBody(exchange.body); + return new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]) }); + }; + const ir = await execute(facts['request.webSearch.canonical'], use.webSearch, dispatch); + return move({ ...facts, 'response.webSearch.ir': ir, 'response.usage.billable': billable }); + }, }); -const execute = async (request: WebSearchRequest, runtime: WebSearchRuntime): Promise => { +const execute = async (request: WebSearchRequest, runtime: WebSearchRuntime, dispatch: AlphaSearchDispatcher): Promise => { const { commands, toolName, iterationCount } = request; const parsed = parseWebSearchOperations(commands); if (iterationCount > ITERATION_CAP) { @@ -38,7 +71,7 @@ const execute = async (request: WebSearchRequest, runtime: WebSearchRuntime): Pr 'Error: arguments must be a JSON object with sub-property arrays (search_query[], open[], find[]).', ); } - if (runtime.executeAlpha !== undefined) { + if (runtime.alpha !== undefined) { const first = parsed.ops[0]; let action: OpenAIResponsesWebSearchAction; if (first.kind === 'search') { @@ -53,7 +86,7 @@ const execute = async (request: WebSearchRequest, runtime: WebSearchRuntime): Pr } else { action = { type: 'search', query: Object.keys(commands).join(', ') }; } - return await runtime.executeAlpha(request, action); + return await executeAlphaSearch({ dispatcher: dispatch, sessionId: runtime.alpha.sessionId, commands: request.commands, settings: request.settings, input: request.input, action, signal: runtime.session.signal }); } try { assertLocalWebSearchSupport(commands); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts index db095e1b1c..95ecb6f5ba 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts @@ -1,11 +1,10 @@ -import type { WebSearchRequest } from './facts.ts'; import type { GatewayServices } from '../../../../pipeline/services.ts'; -import type { WebSearchCallIR, WebSearchExecutionSession } from '../../../../tools/web-search/operations.ts'; -import type { OpenAIResponsesWebSearchAction } from '@floway-dev/protocols/openai-responses'; +import type { WebSearchExecutionSession } from '../../../../tools/web-search/operations.ts'; +import type { ModelCandidate } from '@floway-dev/provider'; export interface WebSearchRuntime { readonly session: Omit; - readonly executeAlpha?: (request: WebSearchRequest, action: OpenAIResponsesWebSearchAction) => Promise; + readonly alpha?: { readonly candidate: Promise; readonly sessionId: string }; } export interface SearchServices extends GatewayServices { readonly webSearch: WebSearchRuntime } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts index d508e3e3fd..b8a13fdc50 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts @@ -3,8 +3,7 @@ import { runWebSearchSubRequest } from './search-sub-request.ts'; import { type HostedToolOutputItem, type HostedToolRegistration } from './types.ts'; import { shortId } from '../../../../shared/short-id.ts'; import { truncatePreservingCodePoints } from '../../../shared/text.ts'; -import { executeAlphaSearch } from '../../../tools/web-search/alpha-search/execution.ts'; -import { resolveAlphaSearchDispatcher } from '../../../tools/web-search/alpha-search/upstream.ts'; +import { resolveAlphaSearchCandidate } from '../../../tools/web-search/alpha-search/upstream.ts'; import { loadWebSearchConfig } from '../../../tools/web-search/config.ts'; import { normalizeDomainEntry } from '../../../tools/web-search/domain-normalize.ts'; import { @@ -603,26 +602,20 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga apiKeyId: gatewayCtx.apiKeyId, ...(gatewayCtx.abortSignal !== undefined ? { signal: gatewayCtx.abortSignal } : {}), }; - let executeAlpha: WebSearchRuntime['executeAlpha']; - if (webSearchConfig.passthroughOpenAiSearch.enabled) { - const dispatcher = resolveAlphaSearchDispatcher({ - config: webSearchConfig.passthroughOpenAiSearch, - upstreamIds: gatewayCtx.upstreamIds, - scheduler: gatewayCtx.backgroundScheduler, - runtimeLocation: gatewayCtx.runtimeLocation, - }); - const sessionId = crypto.randomUUID(); - executeAlpha = async (request, action) => await executeAlphaSearch({ - dispatcher: await dispatcher, - sessionId, - commands: request.commands, - settings: request.settings, - input: request.input, - action, - signal: gatewayCtx.abortSignal, - }); - } - const webSearch: WebSearchRuntime = { session, ...(executeAlpha === undefined ? {} : { executeAlpha }) }; + const webSearch: WebSearchRuntime = { + session, + ...(webSearchConfig.passthroughOpenAiSearch.enabled ? { + alpha: { + candidate: resolveAlphaSearchCandidate({ + config: webSearchConfig.passthroughOpenAiSearch, + upstreamIds: gatewayCtx.upstreamIds, + scheduler: gatewayCtx.backgroundScheduler, + runtimeLocation: gatewayCtx.runtimeLocation, + }), + sessionId: crypto.randomUUID(), + }, + } : {}), + }; return { type: 'active', diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts b/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts index 55ea774f96..0511e88074 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts @@ -14,9 +14,10 @@ const refuseStoredItems = ( status: number, message: string, extra: { readonly param: string; readonly code: string | null }, -): Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'> => ({ +): Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'> => ({ 'response.usage.billable': [], 'response.http.headers': [], + 'response.http.body': null, 'response.chat.openaiResponses.streamedUsage': null, 'response.chat.openaiResponses': { status, @@ -49,7 +50,7 @@ export const hydrateStoredItems = ( Fields<'request.chat.openaiResponses' | 'request.chat.openaiResponses.privatePayloads'>, Record, Record, - Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers'>, + Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, ChatServices >({ name: 'hydrateStoredItems', @@ -66,7 +67,7 @@ export const hydrateStoredItems = ( 'response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', - 'response.http.headers', + 'response.http.headers', 'response.http.body', ], }, execute: async (facts, next, use) => { diff --git a/packages/gateway/src/data-plane/chat/openai-responses/meter.ts b/packages/gateway/src/data-plane/chat/openai-responses/meter.ts index cb369de293..b69d336093 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/meter.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/meter.ts @@ -56,6 +56,7 @@ const meterOpenAIResponses = ( if (isOpenAIResponsesTerminalEvent(frame.event)) { sawTerminal = true; failed = frame.event.type === 'response.failed' || frame.event.type === 'error'; + settle({ billable: [billedOpenAIResponsesEntity(identity, reported)], failed }); } yield frame; // The turn is over, so there is nothing further to read. An upstream that holds the diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 6585199b14..901776afaf 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -41,7 +41,7 @@ export const openaiResponsesServePipeline = ( resolveChatCandidates(openaiResponsesNarrowing(() => prepared)), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), - owns: [], + owns: ['response.http.body'], pendingUsage: OPENAI_RESPONSES_STREAMED_USAGE, }), materializeAttempt('request.chat.openaiResponses'), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index a474eeabf3..5c7a27b376 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -13,7 +13,7 @@ import type { Context } from 'hono'; import { type OpenAIResponsesFacts, type OpenAIResponsesServeExit } from './facts.ts'; import { createOpenAIResponsesWsSession, type OpenAIResponsesStatefulStore } from './items/store.ts'; import { openaiResponsesServePipeline } from './pipeline.ts'; -import { openRunDump } from '../../../dump/run-sink.ts'; +import { recordStream, openRunDump } from '../../../dump/run-sink.ts'; import type { RunDump } from '../../../dump/run-sink.ts'; import { apiKeyFromContext, authenticateApiKey, type AuthedContext } from '../../../middleware/auth.ts'; import type { ApiKey } from '../../../repo/types.ts'; @@ -256,6 +256,7 @@ const openOpenAIResponsesWebSocketTurn = ( services: { ...base.services, gateway, + recordProtocolFrames: frames => recordStream(frames, gateway.dump), rememberChatSelection: payloadFor => { materialize = payloadFor; }, chatPayloadFor: selector => { if (materialize === undefined) { diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts index cb2ec05bb9..e01065b4de 100644 --- a/packages/gateway/src/data-plane/chat/prologue.ts +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -10,11 +10,12 @@ import type { ChatServices } from './services.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from './shared/gateway-ctx.ts'; +import { recordStream } from '../../dump/run-sink.ts'; +import type { AuthedContext } from '../../middleware/auth.ts'; import type { ApiKey } from '../../repo/types.ts'; import type { AttemptSelector } from '../pipeline/facts.ts'; import { gatewayCtxOptions, prologueFor, runDumpOf, type Ingress, type Prologue } from '../pipeline/serve.ts'; import type { OpenAIResponsesStatefulStore } from './openai-responses/items/store.ts'; -import type { AuthedContext } from '../../middleware/auth.ts'; import type { ModelCandidate } from '@floway-dev/provider'; export interface ChatPrologue extends Prologue { @@ -48,6 +49,7 @@ export const openChatPrologue = ( services: { ...base.services, gateway, + recordProtocolFrames: frames => recordStream(frames, gateway.dump), rememberChatSelection: payloadFor => { materialize = payloadFor; }, chatPayloadFor: (selector: AttemptSelector) => { if (materialize === undefined) { diff --git a/packages/gateway/src/data-plane/chat/resolve-candidates.ts b/packages/gateway/src/data-plane/chat/resolve-candidates.ts index b969e30c85..79cfffc366 100644 --- a/packages/gateway/src/data-plane/chat/resolve-candidates.ts +++ b/packages/gateway/src/data-plane/chat/resolve-candidates.ts @@ -37,9 +37,9 @@ export interface ChatNarrowing { export const resolveChatCandidates = (narrowing: ChatNarrowing) => defineStage< Slice<'serve.model'>, Slice<'serve.model' | 'serve.candidates'>, - Slice<'response.usage.billable' | 'response.http.headers'>, - Slice<'response.usage.billable' | 'response.http.headers'>, - Slice<'response.usage.billable' | 'response.http.headers'> & Refusal, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'> & Refusal, ChatServices >({ name: 'resolveChatCandidates', @@ -47,7 +47,7 @@ export const resolveChatCandidates = (narrowing: ChatNar request: { needs: ['serve.model'], consumes: [], provides: ['serve.candidates'] }, response: { needs: ['response.usage.billable', 'response.http.headers'], consumes: [], provides: [] }, }, - return: { provides: ['response.usage.billable', 'response.http.headers', ...narrowing.refuses] }, + return: { provides: ['response.usage.billable', 'response.http.headers', 'response.http.body', ...narrowing.refuses] }, execute: async (facts, next, use) => { const model = facts['serve.model']; const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ @@ -65,6 +65,7 @@ export const resolveChatCandidates = (narrowing: ChatNar ...facts, 'response.usage.billable': [], 'response.http.headers': [], + 'response.http.body': null, ...narrowing.refuse(status, message, reason), }); @@ -86,7 +87,7 @@ export const resolveChatCandidates = (narrowing: ChatNar // is owed is part of that live half — affinity materializes it per candidate, and it // carries the client's own state rewritten for the upstream that will see it. use.rememberChatSelection(selection.payloadFor); - use.log.debug('resolved chat candidates', { model, viable: selection.candidates.length, resolved: candidates.length }); + await use.log.debug('resolved chat candidates', { model, viable: selection.candidates.length, resolved: candidates.length }); return await next({ ...facts, 'serve.candidates': move(use.rememberCandidates(selection.candidates)) }); }, }); diff --git a/packages/gateway/src/data-plane/chat/services.ts b/packages/gateway/src/data-plane/chat/services.ts index 4efb2effd0..c3e46d0379 100644 --- a/packages/gateway/src/data-plane/chat/services.ts +++ b/packages/gateway/src/data-plane/chat/services.ts @@ -1,11 +1,11 @@ import type { GatewayServices } from '../pipeline/services.ts'; import type { ChatGatewayCtx } from './shared/gateway-ctx.ts'; import type { AttemptSelector } from '../pipeline/facts.ts'; -import type { ModelCandidate } from '@floway-dev/provider'; +import type { ModelCandidate, ProviderChatServices } from '@floway-dev/provider'; /** What a chat family needs beyond the shared services: the affinity selection this run * made, so the stage that dials can ask for the payload the winning candidate is owed. */ -export interface ChatServices extends GatewayServices { +export interface ChatServices extends GatewayServices, ProviderChatServices { readonly gateway: ChatGatewayCtx; readonly rememberChatSelection: (payloadFor: (candidate: ModelCandidate) => unknown) => void; readonly chatPayloadFor: (selector: AttemptSelector) => unknown; diff --git a/packages/gateway/src/data-plane/chat/shared/client-stream.ts b/packages/gateway/src/data-plane/chat/shared/client-stream.ts index 11cac696a0..96ef8fe711 100644 --- a/packages/gateway/src/data-plane/chat/shared/client-stream.ts +++ b/packages/gateway/src/data-plane/chat/shared/client-stream.ts @@ -1,6 +1,6 @@ import { recordStream, streamReferenceOf, type RunDump } from '../../../dump/run-sink.ts'; import type { StreamOutcome } from '../../pipeline/serve.ts'; -import { defer, type Deferred } from '@floway-dev/pipeline'; +import { defer, setRelease, type Owned, type Deferred } from '@floway-dev/pipeline'; import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; export const framedClientStream = >( @@ -20,14 +20,16 @@ export const framedClientStream = >( const generator = (async function* () { started = true; let failed = false; + let completed = false; try { for await (const frame of source) yield { frame, wire: render(frame) }; + completed = true; } catch (error) { failed = true; dump?.failed(error); for (const frame of errorFrames(error)) yield { frame, wire: render(frame) }; } finally { - settle(failed); + settle(failed || !completed); } })(); // A protocol frame is published only after it has a valid transport representation. @@ -39,7 +41,7 @@ export const framedClientStream = >( rendered: { ...reference, [Symbol.asyncIterator]: () => (async function* () { for await (const packet of packets) yield packet.wire; })() }, failed, release: async () => { - if (!started) settle(false); + if (!started) settle(true); await generator.return(); }, }; @@ -47,3 +49,11 @@ export const framedClientStream = >( export const withClientVerdict = (reading: Deferred | null, failed: Promise): Deferred | null => reading === null ? null : defer(Promise.all([reading, failed]).then(([outcome, clientFailed]) => ({ ...outcome, failed: outcome.failed || clientFailed }))); + +export const bindClientRelease = (facts: object, release: () => Promise): void => { + if (!('response.http.body' in facts) || facts['response.http.body'] === null) return; + const body = facts['response.http.body'] as ReadableStream & Owned; + const drain = setRelease(body, async () => { + try { await drain(); } finally { await release(); } + }); +}; diff --git a/packages/gateway/src/data-plane/chat/shared/provider-answer.ts b/packages/gateway/src/data-plane/chat/shared/provider-answer.ts new file mode 100644 index 0000000000..37c64d58e6 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/provider-answer.ts @@ -0,0 +1,26 @@ +import { providerCalls } from '../../pipeline/provider-usage.ts'; +import { dialFailure, readUpstreamBody, spentBody } from '../../pipeline/upstream-body.ts'; +import type { ChatAnswer } from '../facts.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import type { Facts } from '@floway-dev/pipeline'; +import type { ChatProviderOperation, ModelCandidate, ProviderChatResponse } from '@floway-dev/provider'; + +export const providerChatAnswer = async (candidate: ModelCandidate, back: ProviderChatResponse) => { + const { 'response.http.exchange': exchange, 'response.provider.output': output } = back; + const billable = providerCalls(candidate, back as unknown as Facts); + if (exchange.type === 'transportFailure') return { + answer: dialFailure(exchange.error), status: 502, headers: [], billable, body: null, + }; + if (output !== null) return { + answer: output as ChatAnswer, + status: 'kind' in output ? exchange.status : output.status, + headers: exchange.headers, + billable, + body: 'kind' in output && output.kind === 'stream' ? exchange.body : spentBody(exchange.body), + }; + const body = await readUpstreamBody(exchangeResponse(exchange)); + return { + answer: { status: exchange.status, message: body.text, ...('json' in body ? { body: body.json } : {}) }, + status: exchange.status, headers: exchange.headers, billable, body: spentBody(exchange.body), + }; +}; diff --git a/packages/gateway/src/data-plane/tools/web-search/alpha-search/execution.ts b/packages/gateway/src/data-plane/tools/web-search/alpha-search/execution.ts index 4636bf0223..4daba6de2a 100644 --- a/packages/gateway/src/data-plane/tools/web-search/alpha-search/execution.ts +++ b/packages/gateway/src/data-plane/tools/web-search/alpha-search/execution.ts @@ -60,8 +60,8 @@ export const executeAlphaSearch = async ({ let parsed: unknown; try { parsed = JSON.parse(raw); - } catch { - throw new Error('OpenAI search upstream returned a non-JSON success body'); + } catch (cause) { + throw new Error('OpenAI search upstream returned a non-JSON success body', { cause }); } if (parsed === null || typeof parsed !== 'object' || typeof (parsed as { output?: unknown }).output !== 'string') { throw new Error('OpenAI search upstream response must include an output string'); diff --git a/packages/gateway/src/data-plane/tools/web-search/alpha-search/upstream.ts b/packages/gateway/src/data-plane/tools/web-search/alpha-search/upstream.ts index d63adec713..81b16d1ac5 100644 --- a/packages/gateway/src/data-plane/tools/web-search/alpha-search/upstream.ts +++ b/packages/gateway/src/data-plane/tools/web-search/alpha-search/upstream.ts @@ -1,12 +1,11 @@ import { enumerateModelCandidates } from '../../../providers/resolution.ts'; -import { filterInboundHeadersForProvider } from '../../../shared/inbound-headers.ts'; import type { WebSearchConfig } from '../types.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import { identityWrapUpstreamCall, providerModelOf } from '@floway-dev/provider'; +import type { ModelCandidate } from '@floway-dev/provider'; export type AlphaSearchDispatcher = (body: Record, signal: AbortSignal | undefined, headers: Headers) => Promise; -export const resolveAlphaSearchDispatcher = async ({ +export const resolveAlphaSearchCandidate = async ({ config, upstreamIds, scheduler, @@ -16,7 +15,7 @@ export const resolveAlphaSearchDispatcher = async ({ upstreamIds: readonly string[] | null; scheduler: BackgroundScheduler; runtimeLocation: string; -}): Promise => { +}): Promise => { if (upstreamIds !== null && !upstreamIds.includes(config.upstreamId)) { throw new Error('Selected OpenAI search upstream is outside this API key scope'); } @@ -35,21 +34,5 @@ export const resolveAlphaSearchDispatcher = async ({ throw new Error('Selected upstream does not support OpenAI search passthrough'); } - return async (body, signal, headers) => { - const { model: _callerModel, ...request } = body; - // TODO: pin SearchRequest.id to one provider account when Codex upstreams - // support account pools. The current Codex provider has one active account. - const result = await candidate.provider.instance.callAlphaSearch( - providerModelOf(candidate), - request, - signal, - { - fetcher: candidate.fetcher, - waitUntil: scheduler, - headers: filterInboundHeadersForProvider(headers, candidate.provider), - wrapUpstreamCall: identityWrapUpstreamCall, - }, - ); - return result.response; - }; + return candidate; }; From effb1066689c41d7885653cc42d7dac4c6b0ffd8 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:10:48 +0800 Subject: [PATCH 123/156] fix: distinguish cancelled chat streams from normal EOF Include the scoped abort signal in the meter verdict so a parser that ends gracefully on cancellation cannot settle an unread Chat Completions response as successful. Verified the official HTTP route by blocking its first protocol frame, cancelling the response, checking the actual provider signal before resuming the source, and asserting one usage observation plus one zero-output failure. Nineteen focused lifecycle cases, gateway typecheck, and scoped lint pass. --- .../chat/openai-chat-completions/http_test.ts | 40 ++++++++++++++++++- packages/gateway/src/data-plane/chat/meter.ts | 5 ++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts index 2d23675ae6..07a0f470d8 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts @@ -1,5 +1,5 @@ import { type Context, Hono } from 'hono'; -import { test, vi } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; import type { AuthVars } from '../../../../src/middleware/auth.ts'; @@ -466,3 +466,41 @@ test('POST /v1/chat/completions leaves TTFT absent on a failure before output', const meta = dumpStubs.stored[0]!.record.meta; assertEquals(meta.ttftMs, null); }); + +test('actual HTTP cancellation before the first client frame releases the called provider and settles failure once', async () => { + const repo = installRepo(); + let unblock!: () => void; + const gate = new Promise(resolve => { unblock = resolve; }); + let closed!: () => void; + const stopped = new Promise(resolve => { closed = resolve; }); + let started = false; + let signal: AbortSignal | undefined; + const call = vi.fn(async (_model: unknown, _body: unknown, calledSignal?: AbortSignal): Promise> => { + signal = calledSignal; + return ({ + ok: true, modelKey: 'called-model', events: (async function* () { + started = true; + try { + await gate; + yield eventFrame(makeOpenAIChatCompletionsEvents()[1]!); + } finally { closed(); } + })(), + }); + }); + queueCandidates([makeCandidate({ callOpenAIChatCompletions: call })]); + const response = await makeApp().request('/v1/chat/completions', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'test-model', stream: true, messages: [{ role: 'user', content: 'hello' }] }), + }); + expect(response.status).toBe(200); + expect(started).toBe(true); + if (response.body === null) throw new Error('Streaming response has no client body'); + await response.body.cancel(); + expect(signal?.aborted).toBe(true); + unblock(); + await stopped; + await flushBackground(); + expect(call).toHaveBeenCalledTimes(1); + expect(await repo.usage.listAll()).toMatchObject([{ modelKey: 'called-model', requests: 1 }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, errorsNoOutput: 1, errorsWithOutput: 0 }]); +}); diff --git a/packages/gateway/src/data-plane/chat/meter.ts b/packages/gateway/src/data-plane/chat/meter.ts index ae848629e1..cfc9f15c06 100644 --- a/packages/gateway/src/data-plane/chat/meter.ts +++ b/packages/gateway/src/data-plane/chat/meter.ts @@ -108,7 +108,10 @@ export const meterChatWire = (spec: { return move({ ...back, [spec.answer]: { kind: 'stream' as const, frames }, - [spec.streamedUsage]: prior.length === 0 ? metered.outcome : defer(metered.outcome.then(outcome => ({ ...outcome, billable: [...prior, ...outcome.billable] }))), + [spec.streamedUsage]: prior.length === 0 && use.gateway.abortSignal === undefined ? metered.outcome : defer(metered.outcome.then(outcome => ({ + ...outcome, billable: prior.length === 0 ? outcome.billable : [...prior, ...outcome.billable], + failed: outcome.failed || use.gateway.abortSignal?.aborted === true, + }))), }) as never; }, }); From 8a83bb1902457765ee73c3f7c2100f283da1d4da Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:12:28 +0800 Subject: [PATCH 124/156] test: drive provider behavior through declared stages Replace direct provider method fixtures and private-rule drivers with actual operation pipelines and immutable stage harnesses. Preserve lazy HTTP-body ownership in gateway fixtures, explicit transport failures, protocol-frame recording and Responses action steering. Regression coverage restores native Azure Anthropic endpoints, verifies immutable compression retries and unchanged tool identity, and preserves unmatched Copilot error exchanges. Six package typechecks and scoped lint pass; 80 provider test files and 679 tests pass. --- packages/http/src/http.ts | 2 +- .../provider-azure/__tests__/provider_test.ts | 64 +++++------ packages/provider-azure/src/pipelines.ts | 6 +- .../src/stages/prepare-request.ts | 19 +++- .../__tests__/provider_test.ts | 107 +++++++----------- .../apply-top-level-cache-control_test.ts | 18 +-- .../compress-images_test.ts | 16 +-- .../handle-speed-fast_test.ts | 38 +++---- .../normalize-anthropic-beta_test.ts | 16 +-- .../promote-thinking-display_test.ts | 32 +++--- .../set-claude-agent-headers_test.ts | 18 +-- .../set-compact-headers_test.ts | 30 ++--- .../set-initiator-header_test.ts | 14 +-- .../set-interaction-id-header_test.ts | 22 ++-- .../set-vision-header_test.ts | 12 +- .../strip-cache-control-extensions_test.ts | 12 +- .../strip-eager-input-streaming_test.ts | 17 +++ .../abort-on-tool-argument-whitespace_test.ts | 8 +- .../attach-cache-control-markers_test.ts | 20 ++-- .../compress-images_test.ts | 12 +- .../set-initiator-header_test.ts | 14 +-- .../set-vision-header_test.ts | 10 +- .../abort-on-tool-argument-whitespace_test.ts | 8 +- .../openai-responses/action-pivot_test.ts | 53 ++++----- .../openai-responses/compress-images_test.ts | 32 ++++-- .../fill-empty-namespace-descriptions_test.ts | 8 +- .../force-store-false_test.ts | 12 +- .../openai-responses/item-id-membrane_test.ts | 20 ++-- .../set-initiator-header_test.ts | 20 ++-- .../set-vision-header_test.ts | 10 +- .../strip-image-generation_test.ts | 2 +- packages/provider-copilot/src/pipelines.ts | 6 +- .../rewrite-context-window-error.ts | 17 ++- .../strip-eager-input-streaming.ts | 8 +- .../src/stages/observe-quota.ts | 4 +- .../abort-on-tool-argument-whitespace.ts | 2 +- .../__tests__/ingress-header-rules_test.ts | 19 ++-- .../__tests__/provider_test.ts | 16 +-- packages/provider-custom/src/pipelines.ts | 6 +- .../src/stages/prepare-request.ts | 2 +- .../__tests__/account-probe_test.ts | 9 +- .../__tests__/provider_test.ts | 15 +-- .../__tests__/usage-probe_test.ts | 16 +-- packages/provider-ollama/src/pipelines.ts | 6 +- .../src/stages/prepare-request.ts | 2 +- packages/provider/src/index.ts | 2 +- .../provider/src/stages/decode-response.ts | 11 +- packages/provider/src/stages/observe-call.ts | 15 ++- .../test-utils/src/chat-provider-pipeline.ts | 50 ++++++++ packages/test-utils/src/index.ts | 3 + packages/test-utils/src/provider-stage.ts | 67 +++++++++++ .../src/stub-chat-provider-pipeline.ts | 84 ++++++++++++++ 52 files changed, 608 insertions(+), 424 deletions(-) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/apply-top-level-cache-control_test.ts (83%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/compress-images_test.ts (93%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/handle-speed-fast_test.ts (69%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/normalize-anthropic-beta_test.ts (76%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/promote-thinking-display_test.ts (78%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/set-claude-agent-headers_test.ts (81%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/set-compact-headers_test.ts (85%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/set-initiator-header_test.ts (80%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/set-interaction-id-header_test.ts (79%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/set-vision-header_test.ts (83%) rename packages/provider-copilot/__tests__/{interceptors => stages}/anthropic-messages/strip-cache-control-extensions_test.ts (87%) create mode 100644 packages/provider-copilot/__tests__/stages/anthropic-messages/strip-eager-input-streaming_test.ts rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts (93%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-chat-completions/attach-cache-control-markers_test.ts (82%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-chat-completions/compress-images_test.ts (89%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-chat-completions/set-initiator-header_test.ts (80%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-chat-completions/set-vision-header_test.ts (80%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/abort-on-tool-argument-whitespace_test.ts (91%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/action-pivot_test.ts (56%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/compress-images_test.ts (82%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/fill-empty-namespace-descriptions_test.ts (80%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/force-store-false_test.ts (76%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/item-id-membrane_test.ts (97%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/set-initiator-header_test.ts (82%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/set-vision-header_test.ts (84%) rename packages/provider-copilot/__tests__/{interceptors => stages}/openai-responses/strip-image-generation_test.ts (98%) create mode 100644 packages/test-utils/src/chat-provider-pipeline.ts create mode 100644 packages/test-utils/src/provider-stage.ts create mode 100644 packages/test-utils/src/stub-chat-provider-pipeline.ts diff --git a/packages/http/src/http.ts b/packages/http/src/http.ts index 1da37bed01..0b1efa5f86 100644 --- a/packages/http/src/http.ts +++ b/packages/http/src/http.ts @@ -24,7 +24,7 @@ export const http = defineStage = {}): UpstreamRecord => { const config = { @@ -120,8 +120,8 @@ test('createAzureProvider sends upstream model ids in OpenAI-shaped request bodi return sseResponse(); }, async () => { - const chat = await instance.instance.callOpenAIChatCompletions(providerModel, { messages: [{ role: 'user', content: 'hello' }] }, undefined, noopUpstreamCallOptions()); - const responses = await instance.instance.callOpenAIResponses(providerModel, { + const chat = await collectChatProviderPipeline(instance, 'openaiChatCompletions', providerModel, { messages: [{ role: 'user', content: 'hello' }] }, undefined, noopUpstreamCallOptions()); + const responses = await collectChatProviderPipeline(instance, 'openaiResponses', providerModel, { input: [{ type: 'additional_tools', role: 'developer', @@ -132,11 +132,11 @@ test('createAzureProvider sends upstream model ids in OpenAI-shaped request bodi tools: [{ type: 'function', name: 'lookup', description: 'Look up a record.', parameters: { type: 'object', properties: {} } }], }], }], - }, 'generate', undefined, noopUpstreamCallOptions()); + }, undefined, noopUpstreamCallOptions()); const embeddings = await callProviderPipeline(instance, 'openaiEmbeddings', providerModel, { input: 'hello' }, undefined, noopUpstreamCallOptions()); - assertEquals(chat.modelKey, 'gpt-prod'); - assertEquals(responses.modelKey, 'gpt-prod'); + assertEquals(chat.facts['response.provider.modelKey'], 'gpt-prod'); + assertEquals(responses.facts['response.provider.modelKey'], 'gpt-prod'); assertEquals(embeddings.modelKey, 'gpt-prod'); }, ); @@ -177,7 +177,7 @@ test('createAzureProvider runs the OpenAI Responses boundary on compact requests }), { status: 200, headers: { 'content-type': 'application/json' } }); }, async () => { - const result = await instance.instance.callOpenAIResponses(providerModel, { + const result = await collectChatProviderPipeline(instance, 'openaiResponsesCompact', providerModel, { input: [{ type: 'additional_tools', role: 'developer', @@ -188,9 +188,9 @@ test('createAzureProvider runs the OpenAI Responses boundary on compact requests tools: [{ type: 'function', name: 'lookup', description: 'Look up a record.', parameters: { type: 'object', properties: {} } }], }], }], - }, 'compact', undefined, noopUpstreamCallOptions()); - assertEquals(result.action, 'compact'); - assertEquals(result.ok, true); + }, undefined, noopUpstreamCallOptions()); + assertEquals(result.facts['request.provider.responsesAction'], 'compact'); + assertEquals((result.output !== null && 'kind' in result.output), true); }, ); @@ -212,21 +212,15 @@ test.each(['generate', 'compact'] as const)('createAzureProvider preserves %s up await withMockedFetch( () => Promise.resolve(upstreamResponse), async () => { - const result = await instance.instance.callOpenAIResponses( - providerModel, - { input: [{ type: 'message', role: 'user', content: 'hello' }] }, - action, - undefined, - noopUpstreamCallOptions(), - ); - assertEquals(result.action, action); - assertEquals(result.ok, false); - if (result.ok) throw new Error('expected upstream error'); - assertEquals(result.modelKey, 'gpt-prod'); - assertEquals(result.response, upstreamResponse); - assertEquals(result.response.status, 418); - assertEquals(result.response.headers.get('x-upstream-error'), action); - assertEquals(await result.response.text(), '{"error":"upstream rejected the request"}'); + const result = await collectChatProviderPipeline(instance, action === 'compact' ? 'openaiResponsesCompact' : 'openaiResponses', providerModel, { input: [{ type: 'message', role: 'user', content: 'hello' }] }, undefined, noopUpstreamCallOptions()); + assertEquals(result.facts['request.provider.responsesAction'], action); + assertEquals((result.output !== null && 'kind' in result.output), false); + if ((result.output !== null && 'kind' in result.output)) throw new Error('expected upstream error'); + assertEquals(result.facts['response.provider.modelKey'], 'gpt-prod'); + if (result.response === null) throw new Error('expected HTTP failure body'); + assertEquals(result.response!.status, 418); + assertEquals(result.response!.headers.get('x-upstream-error'), action); + assertEquals(await result.response!.text(), '{"error":"upstream rejected the request"}'); }, ); }); @@ -271,10 +265,10 @@ test('createAzureProvider supports Azure AI cross-provider models with explicit return sseResponse(); }, async () => { - const chat = await instance.instance.callOpenAIChatCompletions(chatProviderModel, { messages: [{ role: 'user', content: 'hello' }] }, undefined, chatOpts); - const responses = await instance.instance.callOpenAIResponses(openaiResponsesProviderModel, { input: [{ type: 'message', role: 'user', content: 'hello' }] }, 'generate', undefined, openaiResponsesOpts); - assertEquals(chat.modelKey, 'deepseek-v4-pro'); - assertEquals(responses.modelKey, 'gpt-5.4-pro'); + const chat = await collectChatProviderPipeline(instance, 'openaiChatCompletions', chatProviderModel, { messages: [{ role: 'user', content: 'hello' }] }, undefined, chatOpts); + const responses = await collectChatProviderPipeline(instance, 'openaiResponses', openaiResponsesProviderModel, { input: [{ type: 'message', role: 'user', content: 'hello' }] }, undefined, openaiResponsesOpts); + assertEquals(chat.facts['response.provider.modelKey'], 'deepseek-v4-pro'); + assertEquals(responses.facts['response.provider.modelKey'], 'gpt-5.4-pro'); }, ); @@ -333,10 +327,10 @@ test('createAzureProvider supports native Azure Anthropic Messages models', asyn return sseResponse(); }, async () => { - const messages = await instance.instance.callAnthropicMessages(providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, { ...noopUpstreamCallOptions(), anthropicBeta: ['context-1m'] }); - const count = await instance.instance.callAnthropicMessagesCountTokens(providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, { ...noopUpstreamCallOptions(), anthropicBeta: ['context-1m', 'advanced-tool-use'] }); - assertEquals(messages.modelKey, 'claude-prod'); - assertEquals(count.modelKey, 'claude-prod'); + const messages = await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, { ...noopUpstreamCallOptions(), anthropicBeta: ['context-1m'] }); + const count = await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, { ...noopUpstreamCallOptions(), anthropicBeta: ['context-1m', 'advanced-tool-use'] }); + assertEquals(messages.facts['response.provider.modelKey'], 'claude-prod'); + assertEquals(count.facts['response.provider.modelKey'], 'claude-prod'); }, ); @@ -476,7 +470,7 @@ test('createAzureProvider exposes image models and routes generations with api-v assertEquals(models[0].endpoints, { openaiImagesGenerations: {}, openaiImagesEdits: {} }); const result = await callProviderPipeline(provider, 'openaiImagesGenerations', models[0], { prompt: 'hello' }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); - assertEquals(result.response.status, 200); + assertEquals(result.response!.status, 200); }, ); assertEquals(observedUrl, 'https://example.openai.azure.com/openai/v1/images/generations?api-version=preview'); @@ -529,7 +523,7 @@ test('createAzureProvider callOpenAIImagesEdits posts multipart with model repla }], }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); - assertEquals(result.response.status, 200); + assertEquals(result.response!.status, 200); }, ); assertEquals(observedUrl, 'https://example.openai.azure.com/openai/v1/images/edits?api-version=preview'); diff --git a/packages/provider-azure/src/pipelines.ts b/packages/provider-azure/src/pipelines.ts index 776a824d64..c257d86ce5 100644 --- a/packages/provider-azure/src/pipelines.ts +++ b/packages/provider-azure/src/pipelines.ts @@ -3,13 +3,13 @@ import { fillAzureNamespaceDescriptions } from './stages/fill-namespace-descript import { prepareAzureRequest } from './stages/prepare-request.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import { decodeProviderResponse, observeProviderCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction } from '@floway-dev/provider'; import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createAzurePipelines = (config: AzureUpstreamConfig): ProviderPipelines => ({ openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('azure.openaiChatCompletions', [decodeProviderResponse('openaiChatCompletions'), prepareAzureRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), - openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('azure.openaiResponses', [selectProviderResponsesAction('generate'), fillAzureNamespaceDescriptions, decodeProviderResponse('openaiResponses'), prepareAzureRequest(config, 'openaiResponses'), observeProviderCall, http]), - openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('azure.openaiResponsesCompact', [selectProviderResponsesAction('compact'), fillAzureNamespaceDescriptions, decodeProviderResponse('openaiResponsesCompact'), prepareAzureRequest(config, 'openaiResponsesCompact'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('azure.openaiResponses', [selectProviderResponsesAction('generate'), fillAzureNamespaceDescriptions, decodeProviderResponse('openaiResponses'), prepareAzureRequest(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('azure.openaiResponsesCompact', [selectProviderResponsesAction('compact'), fillAzureNamespaceDescriptions, decodeProviderResponse('openaiResponsesCompact'), prepareAzureRequest(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('azure.anthropicMessages', [decodeProviderResponse('anthropicMessages'), prepareAzureRequest(config, 'anthropicMessages'), observeProviderCall, http]), anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('azure.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareAzureRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('azure.openaiCompletions', [prepareAzureRequest(config, 'openaiCompletions'), observeProviderCall, http]), diff --git a/packages/provider-azure/src/stages/prepare-request.ts b/packages/provider-azure/src/stages/prepare-request.ts index 7d8fa3f753..5dbdfec276 100644 --- a/packages/provider-azure/src/stages/prepare-request.ts +++ b/packages/provider-azure/src/stages/prepare-request.ts @@ -1,5 +1,5 @@ import type { AzureUpstreamConfig } from '../config.ts'; -import { azureOpenAiV1BaseUrl } from '../endpoint.ts'; +import { azureAnthropicBaseUrl, azureOpenAiV1BaseUrl } from '../endpoint.ts'; import { type HttpRequestFacts } from '@floway-dev/http/pipeline'; import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import { defineStage, move, secret } from '@floway-dev/pipeline'; @@ -17,8 +17,8 @@ const PATHS: Record, string openaiChatCompletions: '/chat/completions', openaiResponses: '/responses', openaiResponsesCompact: '/responses/compact', - anthropicMessages: '/messages', - anthropicMessagesCountTokens: '/messages/count_tokens', + anthropicMessages: '/v1/messages', + anthropicMessagesCountTokens: '/v1/messages/count_tokens', openaiCompletions: '/completions', openaiEmbeddings: '/embeddings', openaiImagesGenerations: '/images/generations', @@ -39,6 +39,7 @@ export const prepareAzureRequest = (config: AzureUpstr execute: async (facts, next) => { const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; const deployment = (model.providerData as { upstreamModelId: string }).upstreamModelId; + const anthropic = operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens'; let body: HttpBody; let encoding: HttpBodyEncoding = 'json'; let url: URL; @@ -52,7 +53,11 @@ export const prepareAzureRequest = (config: AzureUpstr // https://github.com/Azure/azure-rest-api-specs/blob/928047803788f7377fa003a26ba2bdc2e0fcccc0/specification/cognitiveservices/OpenAI.Inference/routes/audio_transcription.tsp#L19-L49 url.searchParams.set('api-version', '2025-04-01-preview'); } else { - url = new URL(joinBaseAndPath(azureOpenAiV1BaseUrl(config.endpoint), ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') ? PATHS[facts['request.provider.responsesAction'] === 'compact' ? 'openaiResponsesCompact' : 'openaiResponses'] : PATHS[operation as Exclude]))); + const baseUrl = anthropic ? azureAnthropicBaseUrl(config.endpoint) : azureOpenAiV1BaseUrl(config.endpoint); + const path = operation === 'openaiResponses' || operation === 'openaiResponsesCompact' + ? PATHS[facts['request.provider.responsesAction'] === 'compact' ? 'openaiResponsesCompact' : 'openaiResponses'] + : PATHS[operation as Exclude]; + url = new URL(joinBaseAndPath(baseUrl, path)); if (operation === 'openaiImagesEdits') { const prepared = await prepareOpenAIImagesEditsBody(payload as ProviderOperationPayloads['openaiImagesEdits'], deployment); body = prepared.body; @@ -64,9 +69,11 @@ export const prepareAzureRequest = (config: AzureUpstr } if (operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') url.searchParams.append('api-version', 'preview'); } - const base = withHttpContentType([['api-key', secret(config.apiKey)]], body, encoding); + // https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/use-foundry-models + // https://docs.anthropic.com/en/api/versioning + const base = withHttpContentType(anthropic ? [['x-api-key', secret(config.apiKey)], ['anthropic-version', '2023-06-01']] : [['api-key', secret(config.apiKey)]], body, encoding); let headers = mergeHttpHeaders(base, facts['request.http.headers']); - if ('request.provider.anthropicBeta' in facts) { + if (anthropic) { const beta = facts['request.provider.anthropicBeta'] as readonly string[]; headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; diff --git a/packages/provider-copilot/__tests__/provider_test.ts b/packages/provider-copilot/__tests__/provider_test.ts index 616a9a6000..08aa682f84 100644 --- a/packages/provider-copilot/__tests__/provider_test.ts +++ b/packages/provider-copilot/__tests__/provider_test.ts @@ -10,7 +10,7 @@ import { createInMemoryImageProcessor, initImageProcessor } from '@floway-dev/pl import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { UpstreamRecord } from '@floway-dev/provider'; import { directFetcher, initProviderRepo } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertRejects, assertThrows, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, callProviderPipeline, assertEquals, assertRejects, assertThrows, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; const mergeVariantsControl = vi.hoisted<{ override: ((merged: CopilotRawModel[]) => CopilotRawModel[]) | null; @@ -273,7 +273,7 @@ test('Copilot provider owns the claude-* Anthropic Messages capability workaroun assertEquals(providerModel.id, 'claude-haiku-chat-listed'); assertEquals(providerModel.endpoints, { anthropicMessages: {} }); - await provider.callAnthropicMessages(providerModel, { + await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 100, messages: [{ role: 'user', content: 'hello' }], }, undefined, noopAnthropicMessagesUpstreamCallOptions()); @@ -329,7 +329,7 @@ test('Copilot provider selects raw variants that support the target endpoint', a }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - await provider.callOpenAIResponses(providerModel, { + await collectChatProviderPipeline(instance, 'openaiResponses', providerModel, { input: [{ type: 'additional_tools', role: 'developer', @@ -341,7 +341,7 @@ test('Copilot provider selects raw variants that support the target endpoint', a }], }], reasoning: { effort: 'xhigh' }, - }, 'generate', undefined, noopUpstreamCallOptions()); + }, undefined, noopUpstreamCallOptions()); }, ); @@ -400,7 +400,7 @@ test('Copilot provider runs the OpenAI Responses boundary chain on the compact p // service_tier is set so withServiceTierStripped has something to strip; // an input_image is included so withVisionHeaderSet fires; the last // input item is a user message so withInitiatorHeaderSet picks 'user'. - const result = await provider.callOpenAIResponses(providerModel, { + const result = await collectChatProviderPipeline(instance, 'openaiResponsesCompact', providerModel, { input: [ { type: 'additional_tools', @@ -422,11 +422,11 @@ test('Copilot provider runs the OpenAI Responses boundary chain on the compact p }, ], service_tier: 'priority', - }, 'compact', undefined, noopUpstreamCallOptions()); + }, undefined, noopUpstreamCallOptions()); - if (!result.ok) throw new Error('expected ok compaction result'); - if (result.action !== 'compact') throw new Error(`expected compact action tag, got ${result.action}`); - assertEquals(result.result.object, 'response.compaction'); + if (!(result.output !== null && 'kind' in result.output)) throw new Error('expected ok compaction result'); + if (result.output === null || !('kind' in result.output) || result.output.kind !== 'value') throw new Error('expected compact value'); + assertEquals(result.output.body.object, 'response.compaction'); }, ); @@ -590,10 +590,10 @@ test('Copilot provider forces stream=true for streaming endpoints and leaves cou const opts = noopUpstreamCallOptions(); const anthropicMessagesOpts = noopAnthropicMessagesUpstreamCallOptions(); - await provider.callOpenAIChatCompletions(byId.get('gpt-chat')!, { messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); - await provider.callOpenAIResponses(byId.get('gpt-resp')!, { input: [] }, 'generate', undefined, opts); - await provider.callAnthropicMessages(byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); - await provider.callAnthropicMessagesCountTokens(byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); + await collectChatProviderPipeline(instance, 'openaiChatCompletions', byId.get('gpt-chat')!, { messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', byId.get('gpt-resp')!, { input: [] }, undefined, opts); + await collectChatProviderPipeline(instance, 'anthropicMessages', byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); + await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); await callProviderPipeline(instance, 'openaiEmbeddings', byId.get('emb-mini')!, { input: 'hi' }, undefined, opts); }, ); @@ -614,7 +614,7 @@ test('Copilot provider sets copilot-vision-request when an image is nested insid // The vision-detection interceptor runs inside `provider.callAnthropicMessages`, so // it must walk into nested `tool_result.content` to find the image. const driveAnthropicMessages = async (providerModel: Awaited>[number], body: Omit): Promise => { - await provider.callAnthropicMessages(providerModel, body, undefined, noopAnthropicMessagesUpstreamCallOptions()); + await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, body, undefined, noopAnthropicMessagesUpstreamCallOptions()); }; await withMockedFetch( @@ -722,7 +722,7 @@ test('Copilot Anthropic Messages boundary chain does NOT fire on the OpenAI Chat // Even with a Claude-Code-shaped metadata blob, the openai-chat-completions // boundary chain has no Anthropic-Messages-source interceptor, so the // messages-proxy intent must not appear on the wire. - await provider.callOpenAIChatCompletions(providerModel, { + await collectChatProviderPipeline(instance, 'openaiChatCompletions', providerModel, { messages: [{ role: 'user', content: 'hi' }], metadata: { user_id: JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }) }, }, undefined, noopUpstreamCallOptions()); @@ -1013,7 +1013,8 @@ const betaIntent = ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20', 'un test('Copilot consumes Anthropic Messages beta intent before serializing its supported wire subset', async () => { const { copilotUpstream } = await setupCopilotTest(); - const provider = createCopilotProvider(copilotUpstream).instance; + const instance = createCopilotProvider(copilotUpstream); + const provider = instance.instance; let upstreamModel: unknown; let upstreamBeta: string | null = null; @@ -1036,13 +1037,8 @@ test('Copilot consumes Anthropic Messages beta intent before serializing its sup }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - const result = await provider.callAnthropicMessages( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, - undefined, - noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: betaIntent }), - ); - assertEquals(result.modelKey, 'claude-opus-4.6-1m'); + const result = await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: betaIntent })); + assertEquals(result.facts['response.provider.modelKey'], 'claude-opus-4.6-1m'); }, ); @@ -1052,7 +1048,8 @@ test('Copilot consumes Anthropic Messages beta intent before serializing its sup test('Copilot count_tokens consumes and serializes the same Anthropic Messages beta intent', async () => { const { copilotUpstream } = await setupCopilotTest(); - const provider = createCopilotProvider(copilotUpstream).instance; + const instance = createCopilotProvider(copilotUpstream); + const provider = instance.instance; let upstreamModel: unknown; let upstreamBeta: string | null = null; @@ -1073,13 +1070,8 @@ test('Copilot count_tokens consumes and serializes the same Anthropic Messages b }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - const result = await provider.callAnthropicMessagesCountTokens( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, - undefined, - noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: betaIntent }), - ); - assertEquals(result.modelKey, 'claude-opus-4.6-1m'); + const result = await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: betaIntent })); + assertEquals(result.facts['response.provider.modelKey'], 'claude-opus-4.6-1m'); }, ); @@ -1111,17 +1103,12 @@ test('Copilot provider routes speed=fast to the -fast raw variant and stamps usa const [providerModel] = await provider.getProvidedModels(directFetcher); assertEquals(providerModel.id, 'claude-opus-4-6'); - const result = await provider.callAnthropicMessages( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: 'fast' }, - undefined, - noopAnthropicMessagesUpstreamCallOptions(), - ); + const result = await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: 'fast' }, undefined, noopAnthropicMessagesUpstreamCallOptions()); - if (!result.ok) throw new Error(`expected ok stream, got ${JSON.stringify(result.response)}`); + if (!(result.output !== null && 'kind' in result.output)) throw new Error(`expected ok stream, got ${JSON.stringify(result.response)}`); const frames = []; - for await (const frame of result.events) frames.push(frame); + for (const frame of result.frames) frames.push(frame); const messageStart = frames.find(f => f.type === 'event' && f.event.type === 'message_start'); if (messageStart?.type !== 'event' || messageStart.event.type !== 'message_start') { @@ -1135,7 +1122,7 @@ test('Copilot provider routes speed=fast to the -fast raw variant and stamps usa } assertEquals(messageDelta.event.usage?.speed, 'fast'); - assertEquals(result.modelKey, 'claude-opus-4.6-fast'); + assertEquals(result.facts['response.provider.modelKey'], 'claude-opus-4.6-fast'); }, ); @@ -1169,23 +1156,18 @@ test('Copilot provider returns HTTP 400 invalid_request_error when speed=fast hi async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - const result = await provider.callAnthropicMessages( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: 'fast' }, - undefined, - noopAnthropicMessagesUpstreamCallOptions(), - ); + const result = await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: 'fast' }, undefined, noopAnthropicMessagesUpstreamCallOptions()); - if (result.ok) throw new Error('expected 400 error, got ok stream'); - assertEquals(result.response.status, 400); - const body = (await result.response.json()) as { type: string; error: { type: string; message: string } }; + if ((result.output !== null && 'kind' in result.output)) throw new Error('expected 400 error, got ok stream'); + assertEquals(result.response!.status, 400); + const body = (await result.response!.json()) as { type: string; error: { type: string; message: string } }; // Byte-identical to the wire string Anthropic emits on real api.anthropic.com // for the same failure mode — recorded verbatim from a live response in // https://github.com/Yeachan-Heo/gajae-code/blob/main/packages/ai/test/anthropic-fast-mode.test.ts assertEquals(body.type, 'error'); assertEquals(body.error.type, 'invalid_request_error'); assertEquals(body.error.message, "'claude-haiku-4-5' does not support the `speed` parameter."); - assertEquals(result.modelKey, providerModel.id); + assertEquals(result.facts['response.provider.modelKey'], providerModel.id); }, ); @@ -1218,12 +1200,7 @@ test('Copilot provider passes unknown speed values to the upstream verbatim so t // does not own rejecting it and must not strip it either — let // Copilot surface whatever error its strict validator returns. const speedValue = 'priority' as AnthropicMessagesPayload['speed']; - await provider.callAnthropicMessages( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: speedValue }, - undefined, - noopAnthropicMessagesUpstreamCallOptions(), - ); + await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: speedValue }, undefined, noopAnthropicMessagesUpstreamCallOptions()); }, ); @@ -1376,7 +1353,8 @@ test('Copilot chat field: no capabilities → no chat field', async () => { // the SSE body, so a streaming call snapshots at stream start. test('Copilot provider persists the quota snapshot a data-plane response carries', async () => { const harness = await setupCopilotTest(); - const provider = createCopilotProvider(harness.copilotUpstream).instance; + const instance = createCopilotProvider(harness.copilotUpstream); + const provider = instance.instance; await withMockedFetch( request => { @@ -1402,7 +1380,7 @@ test('Copilot provider persists the quota snapshot a data-plane response carries }, async () => { const [model] = await provider.getProvidedModels(directFetcher); - await provider.callOpenAIChatCompletions(model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); + await collectChatProviderPipeline(instance, 'openaiChatCompletions', model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); // The persist is fire-and-forget: on workerd `waitUntil` keeps it alive // past the response, and here it settles on the host event loop. await vi.waitFor(() => { @@ -1424,7 +1402,8 @@ test('Copilot provider persists the quota snapshot a data-plane response carries // no upside, so the slot is left exactly as it was. test('Copilot provider leaves the persisted snapshot alone when a response carries no quota headers', async () => { const harness = await setupCopilotTest(); - const provider = createCopilotProvider(harness.copilotUpstream).instance; + const instance = createCopilotProvider(harness.copilotUpstream); + const provider = instance.instance; await withMockedFetch( request => { @@ -1445,7 +1424,7 @@ test('Copilot provider leaves the persisted snapshot alone when a response carri }, async () => { const [model] = await provider.getProvidedModels(directFetcher); - await provider.callOpenAIChatCompletions(model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); + await collectChatProviderPipeline(instance, 'openaiChatCompletions', model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); }, ); @@ -1501,10 +1480,10 @@ test('Copilot provider merges the -fast raw variant and reaches it through servi const sol = models.find(model => model.id === 'gpt-5.6-sol')!; const opts = noopUpstreamCallOptions(); - await provider.callOpenAIResponses(sol, { input: [] }, 'generate', undefined, opts); - await provider.callOpenAIResponses(sol, { input: [], service_tier: 'priority' }, 'generate', undefined, opts); - await provider.callOpenAIResponses(sol, { input: [], service_tier: 'fast' }, 'generate', undefined, opts); - await provider.callOpenAIResponses(sol, { input: [], service_tier: 'flex' }, 'generate', undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', sol, { input: [] }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', sol, { input: [], service_tier: 'priority' }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', sol, { input: [], service_tier: 'fast' }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', sol, { input: [], service_tier: 'flex' }, undefined, opts); }, ); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts similarity index 83% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts index ea5ba8fee7..f2b363fa09 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withTopLevelCacheControlApplied } from '../../../src/interceptors/anthropic-messages/apply-top-level-cache-control.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesApplyTopLevelCacheControl } from '../../../src/stages/anthropic-messages/apply-top-level-cache-control.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload & { cache_control?: { type: 'ephemeral' } }): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload & { cache_control?: { type: 'ephemeral' } }): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -30,7 +30,7 @@ test('ports top-level cache_control onto the last cacheable block and drops the ], }); - await withTopLevelCacheControlApplied(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[2].content, [{ type: 'text', text: 'second', cache_control: { type: 'ephemeral' } }]); @@ -46,7 +46,7 @@ test('respects an explicit cache_control on the last cacheable block', async () messages: [{ role: 'user', content: [{ type: 'text', text: 'hi', cache_control: explicit }] }], }); - await withTopLevelCacheControlApplied(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'hi', cache_control: explicit }]); @@ -60,7 +60,7 @@ test('promotes string message content so the auto-marker has a block to land on' messages: [{ role: 'user', content: 'plain' }], }); - await withTopLevelCacheControlApplied(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'plain', cache_control: { type: 'ephemeral' } }]); @@ -77,7 +77,7 @@ test('walks past trailing non-cacheable blocks to find the last cacheable one', ], }); - await withTopLevelCacheControlApplied(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[1].content, [{ type: 'thinking', thinking: 'pondering' }]); @@ -92,7 +92,7 @@ test('drops top-level cache_control when no cacheable block exists', async () => messages: [{ role: 'assistant', content: [{ type: 'thinking', thinking: 'no text yet' }] }], }); - await withTopLevelCacheControlApplied(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'thinking', thinking: 'no text yet' }]); @@ -105,7 +105,7 @@ test('no-op when payload has no top-level cache_control', async () => { messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], }); - await withTopLevelCacheControlApplied(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'hi' }]); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts similarity index 93% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts index 808f84d0eb..88d77fa908 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts @@ -1,12 +1,12 @@ import { test } from 'vitest'; -import { withInlineImagesCompressed } from '../../../src/interceptors/anthropic-messages/compress-images.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesCompressImages } from '../../../src/stages/anthropic-messages/compress-images.ts'; import { type ImageDimensions, type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { AnthropicMessagesImageBlock, AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesToolResultBlock } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => @@ -27,7 +27,7 @@ const spyProcessor = (): { processor: ImageProcessor; inputs: Uint8Array[]; targ return { processor, inputs, targets }; }; -const invocation = (payload: AnthropicMessagesPayload, upstreamModelId = 'claude-test'): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload, upstreamModelId = 'claude-test'): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -57,7 +57,7 @@ test('compresses a top-level image block to WebP', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); const block = (ctx.payload.messages[0].content as Array<{ type: string; source?: { media_type: string; data: string } }>)[1]; assertEquals(block.source?.media_type, 'image/webp'); @@ -108,7 +108,7 @@ test('compresses an image nested inside tool_result content', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); const rewrittenContent = ctx.payload.messages[0].content; if (!Array.isArray(rewrittenContent)) throw new Error('expected rewritten user content'); @@ -142,7 +142,7 @@ test('leaves image-free payloads untouched and does not invoke the processor', a }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); assertEquals(inputs.length, 0); assert(ctx.payload === payload); @@ -172,7 +172,7 @@ test('compresses each unique inline image only once when the same base64 data re ], }); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); assertEquals(inputs.length, 2); }); @@ -212,7 +212,7 @@ const capTargetFor = async (upstreamModelId: string, source: ImageDimensions): P }, ], }, upstreamModelId); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); return targets[0]; }; diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts similarity index 69% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts index 65f57302d4..f0d03659e8 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts @@ -1,11 +1,11 @@ import { test } from 'vitest'; -import { withSpeedFast } from '../../../src/interceptors/anthropic-messages/handle-speed-fast.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSpeedFast } from '../../../src/stages/anthropic-messages/handle-speed-fast.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesUsage } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const collect = async (events: AsyncIterable): Promise => { @@ -16,7 +16,7 @@ const collect = async (events: AsyncIterable): Promise => { const baseUsage: AnthropicMessagesUsage = { input_tokens: 10, output_tokens: 0 }; -const makeCtx = (speed?: unknown): AnthropicMessagesBoundaryCtx => ({ +const makeCtx = (speed?: unknown): AnthropicMessagesProbe => ({ payload: { model: 'claude-opus-4.6-fast', messages: [{ role: 'user', content: 'hi' }], @@ -58,38 +58,38 @@ const streamResult = (frames: ProtocolFrame[]): Ex testTelemetryModelIdentity, ); -test('withSpeedFast strips speed=fast from the outbound payload', async () => { +test('copilotAnthropicMessagesSpeedFast strips speed=fast from the outbound payload', async () => { const ctx = makeCtx('fast'); - await withSpeedFast(ctx, () => Promise.resolve(streamResult([]))); + await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([]))); assertEquals('speed' in ctx.payload, false); }); -test('withSpeedFast strips speed=standard (semantically equivalent to omitted)', async () => { +test('copilotAnthropicMessagesSpeedFast strips speed=standard (semantically equivalent to omitted)', async () => { const ctx = makeCtx('standard'); - await withSpeedFast(ctx, () => Promise.resolve(streamResult([]))); + await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([]))); assertEquals('speed' in ctx.payload, false); }); -test('withSpeedFast leaves unknown speed values untouched so the upstream rejects them', async () => { +test('copilotAnthropicMessagesSpeedFast leaves unknown speed values untouched so the upstream rejects them', async () => { // Anthropic returns 400 invalid_request_error on unknown enum values rather // than silently downgrading; the gateway mirrors that by passing the // unknown value through to Copilot, which will surface the same shape of // error. We never invent a fall-through here. const ctx = makeCtx('priority'); - await withSpeedFast(ctx, () => Promise.resolve(streamResult([]))); + await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([]))); assertEquals(ctx.payload.speed, 'priority'); }); -test('withSpeedFast stamps usage.speed=fast on message_start when fast was requested', async () => { +test('copilotAnthropicMessagesSpeedFast stamps usage.speed=fast on message_start when fast was requested', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, () => + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([messageStart(baseUsage), messageStop(), doneFrame()]))); assertEquals(result.type, 'events'); @@ -102,10 +102,10 @@ test('withSpeedFast stamps usage.speed=fast on message_start when fast was reque ]); }); -test('withSpeedFast stamps usage.speed=fast on every message_delta carrying usage', async () => { +test('copilotAnthropicMessagesSpeedFast stamps usage.speed=fast on every message_delta carrying usage', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, () => + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([messageStart(baseUsage), messageDelta(7), messageStop()]))); assertEquals(result.type, 'events'); @@ -122,11 +122,11 @@ test('withSpeedFast stamps usage.speed=fast on every message_delta carrying usag ]); }); -test('withSpeedFast leaves the stream untouched when speed is absent', async () => { +test('copilotAnthropicMessagesSpeedFast leaves the stream untouched when speed is absent', async () => { const ctx = makeCtx(); const frames = [messageStart(baseUsage), messageDelta(3), messageStop()]; - const result = await withSpeedFast(ctx, () => Promise.resolve(streamResult(frames))); + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult(frames))); assertEquals(result.type, 'events'); if (result.type !== 'events') throw new Error('expected events'); @@ -134,21 +134,21 @@ test('withSpeedFast leaves the stream untouched when speed is absent', async () assertEquals(await collect(result.events), frames); }); -test('withSpeedFast leaves the stream untouched when speed=standard (no fast intent to stamp)', async () => { +test('copilotAnthropicMessagesSpeedFast leaves the stream untouched when speed=standard (no fast intent to stamp)', async () => { const ctx = makeCtx('standard'); const frames = [messageStart(baseUsage), messageStop()]; - const result = await withSpeedFast(ctx, () => Promise.resolve(streamResult(frames))); + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult(frames))); assertEquals(result.type, 'events'); if (result.type !== 'events') throw new Error('expected events'); assertEquals(await collect(result.events), frames); }); -test('withSpeedFast surfaces non-events results (api-error / internal-error) verbatim', async () => { +test('copilotAnthropicMessagesSpeedFast surfaces non-events results (api-error / internal-error) verbatim', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, () => + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve({ type: 'internal-error', status: 502, diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts similarity index 76% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts index 9a54c4fbb6..1549c1be03 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts @@ -1,10 +1,10 @@ import { test } from 'vitest'; -import { withAnthropicBetaNormalized } from '../../../src/interceptors/anthropic-messages/normalize-anthropic-beta.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesNormalizeAnthropicBeta } from '../../../src/stages/anthropic-messages/normalize-anthropic-beta.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { eventResult, type ExecuteResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => @@ -13,7 +13,7 @@ const okEvents = (): Promise ({ +): AnthropicMessagesProbe => ({ payload: payload as AnthropicMessagesPayload, headers: new Headers(), anthropicBeta, @@ -33,7 +33,7 @@ test('keeps only supported caller beta values in first-seen order', async () => 'advanced-tool-use-2025-11-20', 'context-1m-2025-08-07', ]); - await withAnthropicBetaNormalized(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, ctx, okEvents); assertEquals(ctx.anthropicBeta, ['advanced-tool-use-2025-11-20']); }); @@ -41,15 +41,15 @@ test('synthesizes interleaved thinking only when the caller supplied no beta int const payload = { ...baseBody, thinking: { type: 'enabled' as const, budget_tokens: 1024 } }; const silent = invocation(payload); const explicit = invocation(payload, ['context-1m-2025-08-07']); - await withAnthropicBetaNormalized(silent, okEvents); - await withAnthropicBetaNormalized(explicit, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, silent, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, explicit, okEvents); assertEquals(silent.anthropicBeta, ['interleaved-thinking-2025-05-14']); assertEquals(explicit.anthropicBeta, []); }); test('does not synthesize interleaved thinking for adaptive thinking', async () => { const ctx = invocation({ ...baseBody, thinking: { type: 'adaptive', budget_tokens: 1024 } }); - await withAnthropicBetaNormalized(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, ctx, okEvents); assertEquals(ctx.anthropicBeta, []); }); @@ -58,7 +58,7 @@ test('pairs context management with its required beta token', async () => { { ...baseBody, context_management: { edits: [] } }, ['interleaved-thinking-2025-05-14'], ); - await withAnthropicBetaNormalized(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, ctx, okEvents); assertEquals(ctx.anthropicBeta, [ 'interleaved-thinking-2025-05-14', 'context-management-2025-06-27', diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts similarity index 78% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts index 1d64787749..b58438264c 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts @@ -1,11 +1,11 @@ import { test } from 'vitest'; -import { resolveAnthropicMessagesDownstreamThinkingDisplay, withThinkingDisplayPromoted } from '../../../src/interceptors/anthropic-messages/promote-thinking-display.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { resolveAnthropicMessagesDownstreamThinkingDisplay, copilotAnthropicMessagesThinkingDisplay } from '../../../src/stages/anthropic-messages/promote-thinking-display.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const collect = async (events: AsyncIterable): Promise => { @@ -15,11 +15,11 @@ const collect = async (events: AsyncIterable): Promise => { }; const makeCtx = ( - thinking: AnthropicMessagesBoundaryCtx['payload']['thinking'], + thinking: AnthropicMessagesProbe['payload']['thinking'], overrides: { model?: string; } = {}, -): AnthropicMessagesBoundaryCtx => ({ +): AnthropicMessagesProbe => ({ payload: { model: overrides.model ?? 'claude-opus-4.7-1m-internal', messages: [{ role: 'user', content: 'hi' }], @@ -80,10 +80,10 @@ test('resolveAnthropicMessagesDownstreamThinkingDisplay ignores unknown explicit assertEquals(resolveAnthropicMessagesDownstreamThinkingDisplay(ctx), undefined); }); -test('withThinkingDisplayPromoted sends summarized upstream when thinking display is omitted', async () => { +test('copilotAnthropicMessagesThinkingDisplay sends summarized upstream when thinking display is omitted', async () => { const ctx = makeCtx({ type: 'adaptive' }); - await withThinkingDisplayPromoted(ctx, () => + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, ctx, () => Promise.resolve({ type: 'internal-error', status: 418, @@ -99,41 +99,41 @@ test('withThinkingDisplayPromoted sends summarized upstream when thinking displa assertEquals(ctx.payload.thinking?.display, 'summarized'); }); -test('withThinkingDisplayPromoted overrides omitted but preserves full', async () => { +test('copilotAnthropicMessagesThinkingDisplay overrides omitted but preserves full', async () => { const omittedCtx = makeCtx({ type: 'adaptive', display: 'omitted' }); const fullCtx = makeCtx({ type: 'adaptive', display: 'full' }); - await withThinkingDisplayPromoted(omittedCtx, okEvents); - await withThinkingDisplayPromoted(fullCtx, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, omittedCtx, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, fullCtx, okEvents); assertEquals(omittedCtx.payload.thinking?.display, 'summarized'); assertEquals(fullCtx.payload.thinking?.display, 'full'); }); -test('withThinkingDisplayPromoted leaves disabled or absent thinking untouched', async () => { +test('copilotAnthropicMessagesThinkingDisplay leaves disabled or absent thinking untouched', async () => { const disabledCtx = makeCtx({ type: 'disabled' }); const absentCtx = makeCtx(undefined); - await withThinkingDisplayPromoted(disabledCtx, okEvents); - await withThinkingDisplayPromoted(absentCtx, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, disabledCtx, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, absentCtx, okEvents); assertEquals(disabledCtx.payload.thinking, { type: 'disabled' }); assertEquals(absentCtx.payload.thinking, undefined); }); -test('withThinkingDisplayPromoted leaves unknown display values for upstream validation', async () => { +test('copilotAnthropicMessagesThinkingDisplay leaves unknown display values for upstream validation', async () => { const ctx = makeCtx({ type: 'adaptive' }); (ctx.payload.thinking as { display?: unknown }).display = 'omit'; - await withThinkingDisplayPromoted(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, ctx, okEvents); assertEquals((ctx.payload.thinking as { display?: unknown }).display, 'omit'); }); -test('withThinkingDisplayPromoted simulates omitted display on protocol events', async () => { +test('copilotAnthropicMessagesThinkingDisplay simulates omitted display on protocol events', async () => { const ctx = makeCtx({ type: 'adaptive' }); - const result = await withThinkingDisplayPromoted(ctx, () => + const result = await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, ctx, () => Promise.resolve( eventResult( (async function* (): AsyncGenerator> { diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts similarity index 81% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts index c1a6bb751c..820024ead1 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts @@ -1,18 +1,18 @@ import { test } from 'vitest'; import { CLAUDE_AGENT_USER_AGENT } from '../../../src/auth.ts'; -import { withClaudeAgentHeadersSet } from '../../../src/interceptors/anthropic-messages/set-claude-agent-headers.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetClaudeAgentHeaders } from '../../../src/stages/anthropic-messages/set-claude-agent-headers.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -29,7 +29,7 @@ const basePayload = (userId: string | undefined): AnthropicMessagesPayload => ({ test('Claude agent headers set for the legacy fingerprint with both halves', async () => { const ctx = invocation(basePayload('user_acct-1_account__session_sess-1')); - await withClaudeAgentHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'messages-proxy'); assertEquals(ctx.headers.get('openai-intent'), 'messages-proxy'); @@ -42,7 +42,7 @@ test('Claude agent headers set for the legacy fingerprint with both halves', asy test('Claude agent headers set for the JSON fingerprint with device_id + session_id', async () => { const ctx = invocation(basePayload(JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }))); - await withClaudeAgentHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.get('user-agent'), CLAUDE_AGENT_USER_AGENT); assertEquals(ctx.headers.get('copilot-integration-id'), ''); @@ -51,7 +51,7 @@ test('Claude agent headers set for the JSON fingerprint with device_id + session test('Claude agent headers absent when session_id is missing', async () => { const ctx = invocation(basePayload(JSON.stringify({ device_id: 'dev-1' }))); - await withClaudeAgentHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -60,7 +60,7 @@ test('Claude agent headers absent when session_id is missing', async () => { test('Claude agent headers absent when safety identifier is missing', async () => { const ctx = invocation(basePayload(JSON.stringify({ session_id: 'sess-only' }))); - await withClaudeAgentHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -69,7 +69,7 @@ test('Claude agent headers absent when safety identifier is missing', async () = test('Claude agent headers absent when metadata is not provided', async () => { const ctx = invocation(basePayload(undefined)); - await withClaudeAgentHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -78,7 +78,7 @@ test('Claude agent headers absent when metadata is not provided', async () => { test('Claude agent headers skipped on claude-opus-4-8 even with full fingerprint', async () => { const ctx = invocation({ ...basePayload(JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' })), model: 'claude-opus-4-8' }); - await withClaudeAgentHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('openai-intent'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts similarity index 85% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts index 3441fb11e9..fb85af9cf5 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withCompactHeadersSet } from '../../../src/interceptors/anthropic-messages/set-compact-headers.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetCompactHeaders } from '../../../src/stages/anthropic-messages/set-compact-headers.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -30,7 +30,7 @@ test('Compact headers set when the last user message carries all three markers', messages: [{ role: 'user', content: COMPACT_LAST_MESSAGE_TEXT }], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); @@ -52,7 +52,7 @@ test('Compact headers set from a multi-block last message that joins to the full ], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); }); @@ -65,7 +65,7 @@ test('Compact headers set when the system prompt starts with a compact summariza messages: [{ role: 'user', content: 'go ahead' }], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); @@ -82,7 +82,7 @@ test('Compact headers set when an array system prompt contains a compact prefix messages: [{ role: 'user', content: 'hi' }], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); }); @@ -94,7 +94,7 @@ test('Compact headers absent when only the text-only guard is present (other mar messages: [{ role: 'user', content: 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.' }], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -107,7 +107,7 @@ test('Compact headers absent for an ordinary user turn', async () => { messages: [{ role: 'user', content: 'hello there' }], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -128,7 +128,7 @@ test('Compact headers absent when the last message is assistant-role with compac ], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -150,7 +150,7 @@ test('Compact headers absent when the last assistant message is a multi-block co ], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -169,7 +169,7 @@ test('Auto-continue absent when the assistant role carries the resume prompt ver ], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -187,7 +187,7 @@ test('Auto-continue marks Claude Code resume prompts with x-initiator: agent onl ], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -205,7 +205,7 @@ test('Auto-continue marks OpenCode primary continuation prompts with x-initiator ], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -223,7 +223,7 @@ test('Auto-continue marks OpenCode media-eviction continuation prompts with x-in ], }); - await withCompactHeadersSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts similarity index 80% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts index c36fdebbf7..f1fb671c0a 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withInitiatorHeaderSet } from '../../../src/interceptors/anthropic-messages/set-initiator-header.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetInitiatorHeader } from '../../../src/stages/anthropic-messages/set-initiator-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -25,7 +25,7 @@ test('Anthropic Messages initiator is user when the last message is a plain user messages: [{ role: 'user', content: 'hello' }], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -45,7 +45,7 @@ test('Anthropic Messages initiator is user when the last user turn mixes text an ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -64,7 +64,7 @@ test('Anthropic Messages initiator is agent when the last user turn is entirely ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -79,7 +79,7 @@ test('Anthropic Messages initiator is agent when the final message is from the a ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts similarity index 79% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts index d160861b28..d6e82b7613 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts @@ -1,18 +1,18 @@ import { parse, validate, version } from 'uuid'; import { test } from 'vitest'; -import { withInteractionIdHeaderSet } from '../../../src/interceptors/anthropic-messages/set-interaction-id-header.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetInteractionIdHeader } from '../../../src/stages/anthropic-messages/set-interaction-id-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -37,7 +37,7 @@ const SESS_ALONE_UUID = '19be0c9e-9a65-43eb-be80-efb2770d4510'; test('Interaction-id forwarded as a SHA-256 UUID for the legacy fingerprint with both halves', async () => { const ctx = invocation(payloadWith('user_acct-1_account__session_sess-legacy')); - await withInteractionIdHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_LEGACY_UUID); }); @@ -45,7 +45,7 @@ test('Interaction-id forwarded as a SHA-256 UUID for the legacy fingerprint with test('Interaction-id forwarded as a SHA-256 UUID for the JSON fingerprint carrying session_id', async () => { const ctx = invocation(payloadWith(JSON.stringify({ device_id: 'dev', session_id: 'sess-json' }))); - await withInteractionIdHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_JSON_UUID); }); @@ -55,7 +55,7 @@ test('Interaction-id forwarded as a SHA-256 UUID even when only the sessionId ha // device_id / account_uuid; we still want trace correlation in that case. const ctx = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); - await withInteractionIdHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_ALONE_UUID); }); @@ -64,8 +64,8 @@ test('Interaction-id hashing is deterministic across repeated invocations', asyn const first = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); const second = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); - await withInteractionIdHeaderSet(first, okEvents); - await withInteractionIdHeaderSet(second, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, first, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, second, okEvents); assertEquals(first.headers.get('x-interaction-id'), second.headers.get('x-interaction-id')); }); @@ -73,7 +73,7 @@ test('Interaction-id hashing is deterministic across repeated invocations', asyn test('Interaction-id has RFC UUIDv4 version and variant bits', async () => { const ctx = invocation(payloadWith(JSON.stringify({ session_id: 'sess-shape-check' }))); - await withInteractionIdHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); const value = ctx.headers.get('x-interaction-id'); assert(value !== null && validate(value)); @@ -85,7 +85,7 @@ test('Interaction-id has RFC UUIDv4 version and variant bits', async () => { test('Interaction-id absent when metadata.user_id is missing', async () => { const ctx = invocation(payloadWith(undefined)); - await withInteractionIdHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.has('x-interaction-id'), false); }); @@ -93,7 +93,7 @@ test('Interaction-id absent when metadata.user_id is missing', async () => { test('Interaction-id absent when metadata.user_id carries no session marker', async () => { const ctx = invocation(payloadWith('user_acct-1_account')); - await withInteractionIdHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.has('x-interaction-id'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts similarity index 83% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts index 3a24e133c5..c186aba4dd 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withVisionHeaderSet } from '../../../src/interceptors/anthropic-messages/set-vision-header.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetVisionHeader } from '../../../src/stages/anthropic-messages/set-vision-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -33,7 +33,7 @@ test('Anthropic Messages vision header set when a top-level image block is prese ], }); - await withVisionHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -58,7 +58,7 @@ test('Anthropic Messages vision header set when an image is nested inside tool_r ], }); - await withVisionHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -81,7 +81,7 @@ test('Anthropic Messages vision header absent when no image is present', async ( ], }); - await withVisionHeaderSet(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts similarity index 87% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts index 88d1082462..f01b5d1c0f 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withCacheControlExtensionsStripped } from '../../../src/interceptors/anthropic-messages/strip-cache-control-extensions.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesStripCacheControlExtensions } from '../../../src/stages/anthropic-messages/strip-cache-control-extensions.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -50,7 +50,7 @@ test('strips scope and ttl from system, tools, and message content blocks while ], }); - await withCacheControlExtensionsStripped(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesStripCacheControlExtensions, ctx, okEvents); assertEquals(ctx.payload.system, [{ type: 'text', text: 'sys', cache_control: { type: 'ephemeral' } }]); assertEquals(ctx.payload.tools, [{ name: 't', input_schema: { type: 'object' }, cache_control: { type: 'ephemeral' } }]); @@ -72,7 +72,7 @@ test('deletes cache_control entirely if no recognised field survives the strip', messages: [{ role: 'user', content: 'hi' }], }); - await withCacheControlExtensionsStripped(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesStripCacheControlExtensions, ctx, okEvents); assertEquals(ctx.payload.system, [{ type: 'text', text: 'sys' }]); }); @@ -86,7 +86,7 @@ test('no-op when no cache_control is present anywhere', async () => { tools: [{ name: 't', input_schema: { type: 'object' } }], }); - await withCacheControlExtensionsStripped(ctx, okEvents); + await applyProviderStage(copilotAnthropicMessagesStripCacheControlExtensions, ctx, okEvents); assertEquals(ctx.payload.system, 'plain system'); assertEquals(ctx.payload.tools, [{ name: 't', input_schema: { type: 'object' } }]); diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-eager-input-streaming_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-eager-input-streaming_test.ts new file mode 100644 index 0000000000..e00c540901 --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-eager-input-streaming_test.ts @@ -0,0 +1,17 @@ +import { expect, test } from 'vitest'; + +import { copilotAnthropicMessagesStripEagerInputStreaming } from '../../../src/stages/anthropic-messages/strip-eager-input-streaming.ts'; +import { applyProviderStage, stubProviderModel } from '@floway-dev/test-utils'; + +test('removes eager_input_streaming while preserving unaffected tool identity and the immutable input', async () => { + const kept = { name: 'kept', input_schema: { type: 'object' as const, properties: {} } }; + const removed = { name: 'changed', input_schema: { type: 'object' as const, properties: {} }, eager_input_streaming: true }; + const payload = { model: 'claude', max_tokens: 16, messages: [], tools: [kept, removed] }; + const probe = { payload, headers: new Headers(), anthropicBeta: [], model: stubProviderModel() }; + await applyProviderStage(copilotAnthropicMessagesStripEagerInputStreaming, probe, async () => {}); + expect(probe.payload.tools[0]).toBe(kept); + expect(probe.payload.tools[1]).not.toBe(removed); + expect(probe.payload.tools[1]).not.toHaveProperty('eager_input_streaming'); + expect(payload.tools[1]).toBe(removed); + expect(removed.eager_input_streaming).toBe(true); +}); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts similarity index 93% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts index ae8695d70f..5a347e0542 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts @@ -1,15 +1,15 @@ import { test } from 'vitest'; -import { withToolArgumentWhitespaceAborted } from '../../../src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/interceptors/shared/whitespace-overflow.ts'; +import { copilotOpenAIChatCompletionsAbortToolWhitespace } from '../../../src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; import { assert, assertEquals, assertStringIncludes, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const invocation = (): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (): OpenAIChatCompletionsProbe => ({ payload: { model: 'test-model', messages: [] }, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -32,7 +32,7 @@ const collect = async (result: ExecuteResult[]): Promise[]> => { - const result = await withToolArgumentWhitespaceAborted(invocation(), () => + const result = await applyProviderStage(copilotOpenAIChatCompletionsAbortToolWhitespace, invocation(), () => Promise.resolve( eventResult( (async function* () { diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts similarity index 82% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts index d0a02d4c89..edd563e987 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { type CopilotCacheableMessage, withCacheControlMarkersAttached } from '../../../src/interceptors/openai-chat-completions/attach-cache-control-markers.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; +import { type CopilotCacheableMessage, copilotOpenAIChatCompletionsAttachCacheControlMarkers } from '../../../src/stages/openai-chat-completions/attach-cache-control-markers.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; import { assert, assertEquals, assertFalse, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (messages: OpenAIChatCompletionsMessage[]): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (messages: OpenAIChatCompletionsMessage[]): OpenAIChatCompletionsProbe => ({ payload: { model: 'gpt-test', messages }, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -30,7 +30,7 @@ test('OpenAI Chat Completions cache markers attach to first two systems and last { role: 'assistant', content: 'assistant 2' }, ]); - await withCacheControlMarkersAttached(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [0, 1, 4, 5]); for (const index of [0, 1, 4, 5]) { @@ -43,7 +43,7 @@ test('OpenAI Chat Completions cache markers handle a single system message', asy { role: 'system', content: 'only system' }, ]); - await withCacheControlMarkersAttached(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [0]); }); @@ -56,7 +56,7 @@ test('OpenAI Chat Completions cache markers attach to last two non-systems when { role: 'assistant', content: 'assistant 2' }, ]); - await withCacheControlMarkersAttached(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [2, 3]); }); @@ -67,7 +67,7 @@ test('OpenAI Chat Completions cache markers skip empty string content', async () { role: 'user', content: '' }, ]); - await withCacheControlMarkersAttached(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), []); }); @@ -78,7 +78,7 @@ test('OpenAI Chat Completions cache markers skip empty array content', async () { role: 'user', content: [] }, ]); - await withCacheControlMarkersAttached(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), []); }); @@ -93,7 +93,7 @@ test('OpenAI Chat Completions cache markers only mark first two of five systems' { role: 'user', content: 'user 1' }, ]); - await withCacheControlMarkersAttached(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); // First two systems + last (and only) non-system. assertEquals(markedIndexes(ctx.payload.messages), [0, 1, 5]); @@ -109,7 +109,7 @@ test('OpenAI Chat Completions cache markers are independent object instances per { role: 'user', content: 'user 1' }, ]); - await withCacheControlMarkersAttached(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); const a = (ctx.payload.messages[0] as CopilotCacheableMessage).copilot_cache_control; const b = (ctx.payload.messages[1] as CopilotCacheableMessage).copilot_cache_control; diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts similarity index 89% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts index 381ef108e8..62ebbff8cc 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts @@ -1,12 +1,12 @@ import { test } from 'vitest'; -import { withInlineImagesCompressed } from '../../../src/interceptors/openai-chat-completions/compress-images.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; +import { copilotOpenAIChatCompletionsCompressImages } from '../../../src/stages/openai-chat-completions/compress-images.ts'; import { type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => @@ -16,7 +16,7 @@ const fixedProcessor: ImageProcessor = { compressToWebp: () => Promise.resolve(new Uint8Array([1, 2, 3])), }; -const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -45,7 +45,7 @@ test('rewrites a base64 image_url data URL to a WebP data URL', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsCompressImages, ctx, okEvents); assertEquals(imageUrl(ctx.payload), 'data:image/webp;base64,AQID'); assertEquals(imageUrl(payload), 'data:image/png;base64,AAAA'); @@ -77,7 +77,7 @@ test('leaves remote https image references untouched', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsCompressImages, ctx, okEvents); assertEquals(imageUrl(ctx.payload), 'https://example.com/cat.png'); assert(ctx.payload === payload); @@ -112,7 +112,7 @@ test('compresses each unique inline image only once when the same data URL repea ], }); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsCompressImages, ctx, okEvents); assertEquals(calls, 2); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts similarity index 80% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts index 693d2445e9..7b83b2e4af 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withInitiatorHeaderSet } from '../../../src/interceptors/openai-chat-completions/set-initiator-header.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; +import { copilotOpenAIChatCompletionsSetInitiatorHeader } from '../../../src/stages/openai-chat-completions/set-initiator-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -23,7 +23,7 @@ test('OpenAI Chat Completions initiator is user when the last message is from th messages: [{ role: 'user', content: 'hello' }], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -37,7 +37,7 @@ test('OpenAI Chat Completions initiator is agent when the last message is an ass ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -56,7 +56,7 @@ test('OpenAI Chat Completions initiator is agent when the last message is a tool ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -76,7 +76,7 @@ test('OpenAI Chat Completions initiator follows the final user role for a lifted ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts similarity index 80% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts index 4bf5f3bd9d..5becb700c7 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withVisionHeaderSet } from '../../../src/interceptors/openai-chat-completions/set-vision-header.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; +import { copilotOpenAIChatCompletionsSetVisionHeader } from '../../../src/stages/openai-chat-completions/set-vision-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -31,7 +31,7 @@ test('OpenAI Chat Completions vision header set when an image_url content part i ], }); - await withVisionHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -45,7 +45,7 @@ test('OpenAI Chat Completions vision header absent when content is pure text', a ], }); - await withVisionHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts similarity index 91% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts index 4bd4e89279..c14ffe7984 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts @@ -1,14 +1,14 @@ import { test } from 'vitest'; -import { withToolArgumentWhitespaceAborted } from '../../../src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/interceptors/shared/whitespace-overflow.ts'; +import { copilotOpenAIResponsesAbortToolWhitespace } from '../../../src/stages/openai-responses/abort-on-tool-argument-whitespace.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; -const invocation = (): OpenAIResponsesBoundaryCtx => ({ +const invocation = (): OpenAIResponsesProbe => ({ payload: { model: 'test-model', input: [], @@ -44,7 +44,7 @@ const collect = async (result: ProviderOpenAIResponsesResult): Promise[]): Promise[]> => { - const result = await withToolArgumentWhitespaceAborted(invocation(), () => + const result = await applyProviderStage(copilotOpenAIResponsesAbortToolWhitespace, invocation(), () => Promise.resolve({ action: 'generate', ok: true, diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/action-pivot_test.ts similarity index 56% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/action-pivot_test.ts index 542d87b503..55daa65fcb 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/action-pivot_test.ts @@ -1,42 +1,31 @@ import { test, vi } from 'vitest'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; import { assertEquals } from '@floway-dev/test-utils'; -// `provider.callOpenAIResponses` runs the boundary chain and the terminal -// switches on `ctx.action`, not on the closure-captured `action` parameter -// — so an interceptor that flips `ctx.action` mid-chain reroutes dispatch. -// To prove that contract end-to-end, swap the boundary chain for one -// containing a pivot interceptor (compact → generate) via `vi.mock`, -// then drive `provider.callOpenAIResponses(model, body, 'compact', ...)`. The -// observed wire request must be the streaming /responses shape -// (stream:true, no `compaction_trigger`), and the typed result must -// surface as the `action: 'generate'` variant. -const pivotCompactToGenerate: Interceptor = async (ctx, run) => { - ctx.action = 'generate'; - return await run(); -}; +type Facts = { 'request.provider.payload': CanonicalOpenAIResponsesPayload; 'request.provider.responsesAction': 'generate' | 'compact' }; -vi.mock('../../../src/interceptors/openai-responses/index.ts', async () => { - const original = await vi.importActual('../../../src/interceptors/openai-responses/index.ts'); - return { - ...original, - COPILOT_OPENAI_RESPONSES_BOUNDARY: [...original.COPILOT_OPENAI_RESPONSES_BOUNDARY, pivotCompactToGenerate], - }; -}); +vi.mock('../../../src/stages/openai-responses/force-store-false.ts', () => ({ + copilotOpenAIResponsesForceStoreFalse: defineStage({ + name: 'pivotCompactToGenerate', + through: { + request: { needs: ['request.provider.payload', 'request.provider.responsesAction'], consumes: ['request.provider.payload', 'request.provider.responsesAction'], provides: ['request.provider.payload', 'request.provider.responsesAction'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': { ...facts['request.provider.payload'], store: false }, 'request.provider.responsesAction': 'generate' })) }), + }), +})); -// Imports below MUST follow the vi.mock so the provider module resolves -// against the mocked chain on first import. +// Resolve the provider assembly after installing the request-stage fixture. const { clearInProcessCopilotTokenCache } = await import('../../../src/auth.ts'); const { createCopilotProvider } = await import('../../../src/provider.ts'); const { createInMemoryImageProcessor, initImageProcessor } = await import('@floway-dev/platform'); const { directFetcher, initProviderRepo } = await import('@floway-dev/provider'); -const { jsonResponse, noopUpstreamCallOptions, sseResponse, withMockedFetch } = await import('@floway-dev/test-utils'); +const { collectChatProviderPipeline, jsonResponse, noopUpstreamCallOptions, sseResponse, withMockedFetch } = await import('@floway-dev/test-utils'); type UpstreamRecord = import('@floway-dev/provider').UpstreamRecord; -test('Copilot provider terminal dispatches on post-chain ctx.action (interceptor flip compact→generate routes to the streaming generate path)', async () => { +test('Copilot dispatch and decoding follow the action changed by a stage (compact→generate)', async () => { const upstream: UpstreamRecord = { id: 'up_copilot_pivot', kind: 'copilot', @@ -98,17 +87,15 @@ test('Copilot provider terminal dispatches on post-chain ctx.action (interceptor }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - const result = await provider.callOpenAIResponses(providerModel, { + const result = await collectChatProviderPipeline(instance, 'openaiResponsesCompact', providerModel, { input: [{ type: 'message', role: 'user', content: 'hi' }], - }, 'compact', undefined, noopUpstreamCallOptions()); - if (!result.ok) throw new Error('expected ok result'); - if (result.action !== 'generate') throw new Error(`expected generate variant after pivot, got ${result.action}`); + }, undefined, noopUpstreamCallOptions()); + if (result.output === null || !('kind' in result.output) || result.output.kind !== 'stream') throw new Error('expected generate stream after action pivot'); + assertEquals(result.facts['response.provider.responsesAction'], 'generate'); }, ); if (!openaiResponsesBody) throw new Error('expected /responses to be hit'); - // Stream-true wire shape proves the terminal took the generate branch, - // not the synth-via-trigger compact branch. assertEquals(openaiResponsesBody.stream, true); const wireInput = openaiResponsesBody.input as Array<{ type: string }>; assertEquals(wireInput.some(item => item.type === 'compaction_trigger'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts similarity index 82% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts index 3ef33a8adb..4aa09fa243 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts @@ -1,12 +1,12 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; -import { withInlineImagesCompressed } from '../../../src/interceptors/openai-responses/compress-images.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesCompressImages } from '../../../src/stages/openai-responses/compress-images.ts'; import { type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputImage, OpenAIResponsesInputItem, OpenAIResponsesStreamEvent, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => @@ -16,7 +16,7 @@ const fixedProcessor: ImageProcessor = { compressToWebp: () => Promise.resolve(new Uint8Array([1, 2, 3])), }; -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -53,7 +53,7 @@ test.each(Object.entries(contentContainers))('compresses inline images in %s', a }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); assertEquals(imageUrlOf(ctx.payload.input[0]), 'data:image/webp;base64,AQID'); assertEquals(imageUrlOf(sourceItem), 'data:image/png;base64,AAAA'); @@ -79,7 +79,7 @@ test.each(Object.entries(contentContainers))('leaves remote images in %s untouch }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); assertEquals(imageUrlOf(ctx.payload.input[0]), 'https://example.com/cat.png'); assert(ctx.payload === payload); @@ -125,7 +125,7 @@ test('compresses each unique inline image only once when the same data URL appea ], }); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); // Two unique data URLs across five targets → exactly two compress calls. assertEquals(calls, 2); @@ -149,16 +149,24 @@ test('reuses its compressed image when the same attempt payload is retried', asy }], }); - await withInlineImagesCompressed(ctx, okEvents); - await withInlineImagesCompressed(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); assertEquals(calls, 1); assertEquals(imageUrlOf(ctx.payload.input[0]), 'data:image/webp;base64,AQID'); const item = ctx.payload.input[0]; - if (item.type !== 'message' || !Array.isArray(item.content) || item.content[0]?.type !== 'input_image') throw new Error('expected image content'); - item.content[0].image_url = 'data:image/png;base64,BBBB'; - await withInlineImagesCompressed(ctx, okEvents); + if (item.type !== 'message' || !Array.isArray(item.content)) throw new Error('expected image message'); + const content = item.content; + const image = content[0]; + if (image?.type !== 'input_image') throw new Error('expected image content'); + expect(() => { image.image_url = 'data:image/png;base64,BBBB'; }).toThrow(); + ctx.payload = { + ...ctx.payload, input: ctx.payload.input.map((candidate, index) => index === 0 + ? { ...item, content: content.map((part, partIndex) => partIndex === 0 ? { ...part, image_url: 'data:image/png;base64,BBBB' } : part) } + : candidate), + }; + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); assertEquals(calls, 2); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts similarity index 80% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts index 3f4538e1ec..ee7518a7e6 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts @@ -1,11 +1,11 @@ import { test } from 'vitest'; -import { withEmptyNamespaceDescriptionsFilled } from '../../../src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesFillEmptyNamespaceDescriptions } from '../../../src/stages/openai-responses/fill-empty-namespace-descriptions.ts'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -36,7 +36,7 @@ test('fills Codex OpenAI Responses Lite namespace descriptions before Copilot di }], }); - await withEmptyNamespaceDescriptionsFilled(ctx, async () => {}); + await applyProviderStage(copilotOpenAIResponsesFillEmptyNamespaceDescriptions, ctx, async () => {}); const [item] = ctx.payload.input; if (item?.type !== 'additional_tools') throw new Error('expected additional_tools input'); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts similarity index 76% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts index 8c6dcee197..438636a614 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withStoreForcedFalse } from '../../../src/interceptors/openai-responses/force-store-false.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesForceStoreFalse } from '../../../src/stages/openai-responses/force-store-false.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -21,7 +21,7 @@ const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBo test('forces store:false when the caller requested store:true', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], store: true }); - await withStoreForcedFalse(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesForceStoreFalse, ctx, okEvents); assertEquals(ctx.payload.store, false); }); @@ -29,7 +29,7 @@ test('forces store:false when the caller requested store:true', async () => { test('sets store:false when the caller omitted store', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); - await withStoreForcedFalse(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesForceStoreFalse, ctx, okEvents); assertEquals(ctx.payload.store, false); }); @@ -37,7 +37,7 @@ test('sets store:false when the caller omitted store', async () => { test('leaves an explicit store:false untouched', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], store: false }); - await withStoreForcedFalse(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesForceStoreFalse, ctx, okEvents); assertEquals(ctx.payload.store, false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts similarity index 97% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts index daca6c17e2..65788e90a4 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts @@ -1,14 +1,14 @@ import { expect, test } from 'vitest'; import { unwrapCopilotItemId, wrapCopilotItemId } from '../../../src/interceptors/openai-responses/item-id-carrier.ts'; -import { withCopilotOpenAIResponsesItemIdMembrane } from '../../../src/interceptors/openai-responses/item-id-membrane.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesItemIdMembrane } from '../../../src/stages/openai-responses/item-id-membrane.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { openaiResponsesResultToEvents, type OpenAIResponsesInputItem, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { stubProviderModel } from '@floway-dev/test-utils'; -const invocation = (input: OpenAIResponsesInputItem[] = []): OpenAIResponsesBoundaryCtx => ({ +const invocation = (input: OpenAIResponsesInputItem[] = []): OpenAIResponsesProbe => ({ payload: { model: 'test-model', input, @@ -49,13 +49,13 @@ const collect = async (result: ProviderOpenAIResponsesResult): Promise> | ProtocolFrame[], ctx = invocation(), -): Promise<{ result: ProviderOpenAIResponsesResult; ctx: OpenAIResponsesBoundaryCtx }> => { +): Promise<{ result: ProviderOpenAIResponsesResult; ctx: OpenAIResponsesProbe }> => { const iterable = Symbol.asyncIterator in frames ? frames as AsyncIterable> : (async function* () { yield* frames as ProtocolFrame[]; })(); - const result = await withCopilotOpenAIResponsesItemIdMembrane(ctx, () => Promise.resolve({ - action: 'generate', - ok: true, + const result = await applyProviderStage(copilotOpenAIResponsesItemIdMembrane, ctx, () => Promise.resolve({ + action: 'generate' as const, + ok: true as const, events: iterable, modelKey: 'test-model', })); @@ -404,7 +404,7 @@ test('restores owned blob ids for Copilot input and leaves foreign items unchang ]; const ctx = invocation(input); let wireInput: OpenAIResponsesInputItem[] | undefined; - await withCopilotOpenAIResponsesItemIdMembrane(ctx, () => { + await applyProviderStage(copilotOpenAIResponsesItemIdMembrane, ctx, () => { wireInput = structuredClone(ctx.payload.input); return Promise.resolve({ action: 'generate', @@ -445,7 +445,7 @@ test('rejects conflicting ids carried by one input item', async () => { ], }]); - await expect(withCopilotOpenAIResponsesItemIdMembrane(ctx, () => { + await expect(applyProviderStage(copilotOpenAIResponsesItemIdMembrane, ctx, () => { throw new Error('must not reach upstream'); })).rejects.toThrow(/conflicting upstream ids/); }); @@ -455,7 +455,7 @@ test('normalizes the generated compaction item without touching retained compact { type: 'message', id: 'msg_retained', status: 'completed', role: 'assistant', content: [] }, { type: 'compaction', id: 'cmp_raw', encrypted_content: 'compact state' }, ]); - const result = await withCopilotOpenAIResponsesItemIdMembrane(invocation(), () => Promise.resolve({ + const result = await applyProviderStage(copilotOpenAIResponsesItemIdMembrane, invocation(), () => Promise.resolve({ action: 'compact', ok: true, result: compactResult, diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts similarity index 82% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts index 8af3e3e8d1..50671ad72a 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withInitiatorHeaderSet } from '../../../src/interceptors/openai-responses/set-initiator-header.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesSetInitiatorHeader } from '../../../src/stages/openai-responses/set-initiator-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -30,7 +30,7 @@ test.each(['user', 'system', 'developer'] as const)('OpenAI Responses initiator ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -38,7 +38,7 @@ test.each(['user', 'system', 'developer'] as const)('OpenAI Responses initiator test('OpenAI Responses initiator is user when input is an empty array', async () => { const ctx = invocation({ model: 'gpt-test', input: [] }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -53,7 +53,7 @@ test('OpenAI Responses initiator is user for the role-bearing additional_tools i }], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -82,7 +82,7 @@ test('OpenAI Responses initiator is agent when the last input item is a function ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -110,7 +110,7 @@ test('OpenAI Responses initiator is agent when the last input item is a custom_t ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -128,7 +128,7 @@ test('OpenAI Responses initiator is agent when the last canonical item is reason ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -150,7 +150,7 @@ test('OpenAI Responses initiator is agent when the last input item is an assista ], }); - await withInitiatorHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts similarity index 84% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts index 66d10d45c7..99b66aa8aa 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts @@ -1,17 +1,17 @@ import { test } from 'vitest'; -import { withVisionHeaderSet } from '../../../src/interceptors/openai-responses/set-vision-header.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesSetVisionHeader } from '../../../src/stages/openai-responses/set-vision-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputItem, OpenAIResponsesStreamEvent, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; import type { ExecuteResult } from '@floway-dev/provider'; import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -33,7 +33,7 @@ test.each(Object.entries(contentContainers))('OpenAI Responses vision header det ])], }); - await withVisionHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -44,7 +44,7 @@ test.each(Object.entries(contentContainers))('OpenAI Responses vision header ign input: [wrap([{ type: 'input_text', text: 'plain text only' }])], }); - await withVisionHeaderSet(ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/strip-image-generation_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/strip-image-generation_test.ts similarity index 98% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/strip-image-generation_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/strip-image-generation_test.ts index 421d465910..9517e9bebb 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/strip-image-generation_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/strip-image-generation_test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; -import { stripImageGenerationFromPayload } from '../../../src/interceptors/openai-responses/strip-image-generation.ts'; +import { stripImageGenerationFromPayload } from '../../../src/stages/openai-responses/strip-image-generation.ts'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; import { assert, assertEquals, assertFalse } from '@floway-dev/test-utils'; diff --git a/packages/provider-copilot/src/pipelines.ts b/packages/provider-copilot/src/pipelines.ts index 1fd7c3e642..4aa984e6af 100644 --- a/packages/provider-copilot/src/pipelines.ts +++ b/packages/provider-copilot/src/pipelines.ts @@ -36,13 +36,13 @@ import { reshapeCopilotCompaction } from './stages/reshape-compaction.ts'; import { selectCopilotChatModel } from './stages/select-chat-model.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import { decodeProviderResponse, observeProviderCall, type ProviderOperationRequest, type ProviderOperationResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, type ProviderOperationRequest, type ProviderOperationResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; export const createCopilotPipelines = (auth: CopilotAuth): ProviderPipelines => ({ openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('copilot.openaiChatCompletions', [selectCopilotChatModel('openaiChatCompletions'), authenticateCopilotChat(auth), copilotOpenAIChatCompletionsCompressImages, copilotOpenAIChatCompletionsAbortToolWhitespace, copilotOpenAIChatCompletionsAttachCacheControlMarkers, copilotOpenAIChatCompletionsSetInitiatorHeader, copilotOpenAIChatCompletionsSetVisionHeader, decodeProviderResponse('openaiChatCompletions'), prepareCopilotChatRequest('openaiChatCompletions'), observeCopilotQuota(auth.id), observeProviderCall, http]), - openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('copilot.openaiResponses', [selectCopilotChatModel('openaiResponses'), authenticateCopilotChat(auth), copilotOpenAIResponsesCompressImages, copilotOpenAIResponsesFillEmptyNamespaceDescriptions, copilotOpenAIResponsesStripServiceTier, copilotOpenAIResponsesStripImageGeneration, copilotOpenAIResponsesForceStoreFalse, copilotOpenAIResponsesItemIdMembrane, copilotOpenAIResponsesAbortToolWhitespace, copilotOpenAIResponsesSetVisionHeader, copilotOpenAIResponsesSetInitiatorHeader, decodeProviderResponse('openaiResponses'), prepareCopilotChatRequest('openaiResponses'), observeCopilotQuota(auth.id), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('copilot.openaiResponses', [selectCopilotChatModel('openaiResponses'), authenticateCopilotChat(auth), copilotOpenAIResponsesCompressImages, copilotOpenAIResponsesFillEmptyNamespaceDescriptions, copilotOpenAIResponsesStripServiceTier, copilotOpenAIResponsesStripImageGeneration, copilotOpenAIResponsesForceStoreFalse, copilotOpenAIResponsesItemIdMembrane, copilotOpenAIResponsesAbortToolWhitespace, copilotOpenAIResponsesSetVisionHeader, copilotOpenAIResponsesSetInitiatorHeader, decodeProviderResponse('openaiResponses'), prepareCopilotChatRequest('openaiResponses'), observeCopilotQuota(auth.id), observeProviderResponsesCall, http]), anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('copilot.anthropicMessages', [selectCopilotChatModel('anthropicMessages'), authenticateCopilotChat(auth), rewriteCopilotContextWindowError, copilotAnthropicMessagesSetCompactHeaders, copilotAnthropicMessagesSetClaudeAgentHeaders, copilotAnthropicMessagesSetInteractionIdHeader, copilotAnthropicMessagesCompressImages, copilotAnthropicMessagesSpeedFast, copilotAnthropicMessagesThinkingDisplay, copilotAnthropicMessagesApplyTopLevelCacheControl, copilotAnthropicMessagesStripCacheControlExtensions, copilotAnthropicMessagesStripEagerInputStreaming, copilotAnthropicMessagesSetVisionHeader, copilotAnthropicMessagesSetInitiatorHeader, copilotAnthropicMessagesNormalizeAnthropicBeta, decodeProviderResponse('anthropicMessages'), prepareCopilotChatRequest('anthropicMessages'), observeCopilotQuota(auth.id), observeProviderCall, http]), anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('copilot.anthropicMessagesCountTokens', [selectCopilotChatModel('anthropicMessagesCountTokens'), authenticateCopilotChat(auth), copilotAnthropicMessagesCompressImages, copilotAnthropicMessagesSetVisionHeader, copilotAnthropicMessagesSetInitiatorHeader, copilotAnthropicMessagesNormalizeAnthropicBeta, decodeProviderResponse('anthropicMessagesCountTokens'), prepareCopilotChatRequest('anthropicMessagesCountTokens'), observeCopilotQuota(auth.id), observeProviderCall, http]), - openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('copilot.openaiResponsesCompact', [selectCopilotChatModel('openaiResponsesCompact'), authenticateCopilotChat(auth), copilotOpenAIResponsesCompressImages, copilotOpenAIResponsesFillEmptyNamespaceDescriptions, copilotOpenAIResponsesStripServiceTier, copilotOpenAIResponsesStripImageGeneration, copilotOpenAIResponsesForceStoreFalse, copilotOpenAIResponsesItemIdMembrane, copilotOpenAIResponsesAbortToolWhitespace, copilotOpenAIResponsesSetVisionHeader, copilotOpenAIResponsesSetInitiatorHeader, reshapeCopilotCompaction, decodeProviderResponse('openaiResponsesCompact'), prepareCopilotChatRequest('openaiResponsesCompact'), observeCopilotQuota(auth.id), observeProviderCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('copilot.openaiResponsesCompact', [selectCopilotChatModel('openaiResponsesCompact'), authenticateCopilotChat(auth), copilotOpenAIResponsesCompressImages, copilotOpenAIResponsesFillEmptyNamespaceDescriptions, copilotOpenAIResponsesStripServiceTier, copilotOpenAIResponsesStripImageGeneration, copilotOpenAIResponsesForceStoreFalse, copilotOpenAIResponsesItemIdMembrane, copilotOpenAIResponsesAbortToolWhitespace, copilotOpenAIResponsesSetVisionHeader, copilotOpenAIResponsesSetInitiatorHeader, reshapeCopilotCompaction, decodeProviderResponse('openaiResponsesCompact'), prepareCopilotChatRequest('openaiResponsesCompact'), observeCopilotQuota(auth.id), observeProviderResponsesCall, http]), openaiEmbeddings: compose('copilot.openaiEmbeddings', [authenticateCopilot(auth), prepareCopilotEmbeddings(), observeCopilotQuota(auth.id), observeProviderCall, http]), }); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts b/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts index 82954c6a4b..2c32fb034c 100644 --- a/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts @@ -1,4 +1,4 @@ -import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { exchangeResponse, takeHttpResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, setRelease } from '@floway-dev/pipeline'; import { buildPromptTooLongBody } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderChatResponse } from '@floway-dev/provider'; @@ -7,22 +7,27 @@ export const rewriteCopilotContextWindowError = defineStage { const back = await next(move({ ...facts })); const exchange = back['response.http.exchange']; if (exchange.type !== 'response' || exchange.status >= 200 && exchange.status < 300) return move({ ...back, 'response.copilot.errorBody': null }); - const text = await exchangeResponse(exchange).text(); + const bytes = new Uint8Array(await exchangeResponse(exchange).arrayBuffer()); + const text = new TextDecoder().decode(bytes); if (exchange.body !== null) setRelease(exchange.body, async () => {}); let body: unknown; try { body = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; body = text; } const tooLong = text.includes('Request body is too large for model context window') || text.includes('context_length_exceeded'); - const rewritten = tooLong ? JSON.parse(new TextDecoder().decode(buildPromptTooLongBody())) as unknown : body; + if (!tooLong) { + const replay = takeHttpResponse(new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]) })); + return move({ ...back, 'response.http.exchange': replay, 'response.http.body': replay.body, 'response.copilot.errorBody': body }); + } + const rewritten: unknown = JSON.parse(new TextDecoder().decode(buildPromptTooLongBody())); return move({ ...back, 'response.copilot.errorBody': body, 'response.provider.output': { - status: tooLong ? 400 : exchange.status, - message: tooLong ? 'prompt is too long' : `Upstream returned ${exchange.status}`, + status: 400, + message: 'prompt is too long', body: rewritten, }, }); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts b/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts index 72c3836850..158b58c172 100644 --- a/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts @@ -1,5 +1,6 @@ import type { MessagesFacts } from '../../chat-facts.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import { mapKeepingIdentity, withKeysChanged } from '@floway-dev/protocols/common'; /** * `eager_input_streaming` is a per-tool property in the Anthropic Messages API @@ -23,14 +24,15 @@ export const copilotAnthropicMessagesStripEagerInputStreaming = defineStage { + payload = withKeysChanged(payload, { + tools: mapKeepingIdentity(payload.tools, tool => { + if (!Object.hasOwn(tool, 'eager_input_streaming')) return tool; const { eager_input_streaming: _, ...rest } = tool as typeof tool & { eager_input_streaming?: unknown; }; return rest; }), - }; + }); } return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); diff --git a/packages/provider-copilot/src/stages/observe-quota.ts b/packages/provider-copilot/src/stages/observe-quota.ts index 22cdb1d138..0f709c649a 100644 --- a/packages/provider-copilot/src/stages/observe-quota.ts +++ b/packages/provider-copilot/src/stages/observe-quota.ts @@ -17,8 +17,8 @@ export const observeCopilotQuota = (upstreamId: string) => { const pending: Promise[] = []; if (exchange.type === 'response') { const snapshot = parseCopilotQuotaHeaders(new Headers(exchange.headers.map(([name, value]): [string, string] => [name, value])), new Date()); - if (snapshot !== null) pending.push(putCopilotQuota(upstreamId, snapshot).catch(error => { - use.log.warn('Copilot quota snapshot persistence failed', { upstreamId, error }); + if (snapshot !== null) pending.push(putCopilotQuota(upstreamId, snapshot).catch(async error => { + await use.log.warn('Copilot quota snapshot persistence failed', { upstreamId, error }); })); } return move({ ...back, 'response.copilot.quota': defer(Promise.all(pending)) }); diff --git a/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts index 06d9d041c1..527b9a14ef 100644 --- a/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts +++ b/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts @@ -59,7 +59,7 @@ export const copilotOpenAIResponsesAbortToolWhitespace = defineStage { const messagesBody = { max_tokens: 10, messages: [{ role: 'user' as const, content: 'hi' }] }; await callProviderPipeline(provider, 'alphaSearch', model, { query: 'hi' }, undefined, opts()); - await provider.instance.callOpenAIChatCompletions(model, { messages: [] }, undefined, opts()); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', model, { messages: [] }, undefined, opts()); await callProviderPipeline(provider, 'openaiCompletions', model, { prompt: 'hi' }, undefined, opts()); - await provider.instance.callOpenAIResponses(model, { input: [] }, 'generate', undefined, opts()); - await provider.instance.callOpenAIResponses(model, { input: [] }, 'compact', undefined, opts()); - await provider.instance.callAnthropicMessages(model, messagesBody, undefined, messagesOpts()); - await provider.instance.callAnthropicMessagesCountTokens(model, messagesBody, undefined, messagesOpts()); + await collectChatProviderPipeline(provider, 'openaiResponses', model, { input: [] }, undefined, opts()); + await collectChatProviderPipeline(provider, 'openaiResponsesCompact', model, { input: [] }, undefined, opts()); + await collectChatProviderPipeline(provider, 'anthropicMessages', model, messagesBody, undefined, messagesOpts()); + await collectChatProviderPipeline(provider, 'anthropicMessagesCountTokens', model, messagesBody, undefined, messagesOpts()); await callProviderPipeline(provider, 'openaiEmbeddings', model, { input: 'hi' }, undefined, opts()); await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'hi' }, undefined, opts()); await callProviderPipeline(provider, 'openaiImagesEdits', model, { diff --git a/packages/provider-custom/__tests__/provider_test.ts b/packages/provider-custom/__tests__/provider_test.ts index 31092d1c19..24ff30e84e 100644 --- a/packages/provider-custom/__tests__/provider_test.ts +++ b/packages/provider-custom/__tests__/provider_test.ts @@ -5,7 +5,7 @@ import type { ModelPricing } from '@floway-dev/protocols/common'; import { parseRerankRequest } from '@floway-dev/protocols/rerank'; import type { UpstreamModelConfig, UpstreamRecord } from '@floway-dev/provider'; import { directFetcher } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertExists, assertRejects, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, callProviderPipeline, assertEquals, assertExists, assertRejects, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; interface BuildOptions { ingressHeadersRules?: { key: string; value: string | null }[]; @@ -67,7 +67,7 @@ test('Custom writes every configured header value and passes admitted client val 'x-override': 'client-override', }), }); - await provider.instance.callOpenAIChatCompletions(model, { messages: [] }, undefined, opts); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', model, { messages: [] }, undefined, opts); }, ); @@ -409,10 +409,10 @@ test('Custom provider forces stream=true for streaming endpoints and leaves coun assertExists(model); const opts = noopUpstreamCallOptions(); const anthropicMessagesOpts = noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: ['context-1m', 'advanced-tool-use'] }); - await provider.callOpenAIChatCompletions(model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); - await provider.callOpenAIResponses(model, { input: [] }, 'generate', undefined, opts); - await provider.callAnthropicMessages(model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); - await provider.callAnthropicMessagesCountTokens(model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); + await collectChatProviderPipeline(instance, 'openaiChatCompletions', model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', model, { input: [] }, undefined, opts); + await collectChatProviderPipeline(instance, 'anthropicMessages', model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); + await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); await callProviderPipeline(instance, 'openaiEmbeddings', model, { input: 'hi' }, undefined, opts); }, ); @@ -495,7 +495,7 @@ test('Custom provider callOpenAIImagesGenerations posts JSON with model re-injec const [model] = await provider.instance.getProvidedModels(directFetcher); const result = await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'hi' }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); - assertEquals(result.response.status, 200); + assertEquals(result.response!.status, 200); }, ); assertExists(forwarded); @@ -523,7 +523,7 @@ test('Custom provider callAlphaSearch posts JSON to /v1/alpha/search with the up { id: 'search-session', commands: { search_query: [{ q: 'Floway' }] } }, undefined, noopUpstreamCallOptions()); - assertEquals(result.response.status, 200); + assertEquals(result.response!.status, 200); assertEquals(result.modelKey, 'gpt-search'); }, ); diff --git a/packages/provider-custom/src/pipelines.ts b/packages/provider-custom/src/pipelines.ts index b67c1d6c80..7278a3e650 100644 --- a/packages/provider-custom/src/pipelines.ts +++ b/packages/provider-custom/src/pipelines.ts @@ -2,14 +2,14 @@ import type { CustomUpstreamConfig } from './config.ts'; import { prepareCustomRequest } from './stages/prepare-request.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import { decodeProviderResponse, observeProviderCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction } from '@floway-dev/provider'; import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createCustomPipelines = (config: CustomUpstreamConfig): ProviderPipelines => ({ alphaSearch: compose, ProviderOperationResponse<'alphaSearch'>>('custom.alphaSearch', [prepareCustomRequest(config, 'alphaSearch'), observeProviderCall, http]), openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('custom.openaiChatCompletions', [decodeProviderResponse('openaiChatCompletions'), prepareCustomRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), - openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('custom.openaiResponses', [selectProviderResponsesAction('generate'), decodeProviderResponse('openaiResponses'), prepareCustomRequest(config, 'openaiResponses'), observeProviderCall, http]), - openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('custom.openaiResponsesCompact', [selectProviderResponsesAction('compact'), decodeProviderResponse('openaiResponsesCompact'), prepareCustomRequest(config, 'openaiResponsesCompact'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('custom.openaiResponses', [selectProviderResponsesAction('generate'), decodeProviderResponse('openaiResponses'), prepareCustomRequest(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('custom.openaiResponsesCompact', [selectProviderResponsesAction('compact'), decodeProviderResponse('openaiResponsesCompact'), prepareCustomRequest(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('custom.anthropicMessages', [decodeProviderResponse('anthropicMessages'), prepareCustomRequest(config, 'anthropicMessages'), observeProviderCall, http]), anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('custom.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareCustomRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('custom.openaiCompletions', [prepareCustomRequest(config, 'openaiCompletions'), observeProviderCall, http]), diff --git a/packages/provider-custom/src/stages/prepare-request.ts b/packages/provider-custom/src/stages/prepare-request.ts index cb108b6ce5..777e660dc6 100644 --- a/packages/provider-custom/src/stages/prepare-request.ts +++ b/packages/provider-custom/src/stages/prepare-request.ts @@ -108,7 +108,7 @@ export const prepareCustomRequest = (config: Custom } base = withHttpContentType(base, body, encoding); let headers = mergeHttpHeaders(base, resolvedHeaders(config, facts['request.http.headers'])); - if ('request.provider.anthropicBeta' in facts) { + if (operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens') { const beta = facts['request.provider.anthropicBeta'] as readonly string[]; headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; diff --git a/packages/provider-ollama/__tests__/account-probe_test.ts b/packages/provider-ollama/__tests__/account-probe_test.ts index 51582efb7d..9402a865e0 100644 --- a/packages/provider-ollama/__tests__/account-probe_test.ts +++ b/packages/provider-ollama/__tests__/account-probe_test.ts @@ -5,7 +5,7 @@ import { assertOllamaUpstreamRecord } from '../src/config.ts'; import { createOllamaProvider } from '../src/provider.ts'; import { readOllamaUpstreamState } from '../src/state.ts'; import { directFetcher, initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; -import { assertEquals, assertRejects, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, assertEquals, assertRejects, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; const UPSTREAM_ID = 'up_ollama_account'; @@ -130,12 +130,7 @@ const callChat = async (record: UpstreamRecord, onCall: (pathname: string) => vo return new Response('data: [DONE]\n\n', { status: 200, headers: { 'content-type': 'text/event-stream' } }); }, async () => { - await provider.instance.callOpenAIChatCompletions( - stubProviderModel({ providerData: 'gpt-oss:120b' }), - { messages: [] }, - undefined, - noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } }), - ); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', stubProviderModel({ providerData: 'gpt-oss:120b' }), { messages: [] }, undefined, noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } })); await Promise.all(pending); }, ); diff --git a/packages/provider-ollama/__tests__/provider_test.ts b/packages/provider-ollama/__tests__/provider_test.ts index 9fd4aead69..d0d3d36a23 100644 --- a/packages/provider-ollama/__tests__/provider_test.ts +++ b/packages/provider-ollama/__tests__/provider_test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { createOllamaProvider } from '../src/provider.ts'; import { initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, testFetcher, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, callProviderPipeline, assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, testFetcher, withMockedFetch } from '@floway-dev/test-utils'; // A cloud upstream writes its usage snapshot after the calls it serves, so the // provider needs a repo to write into wherever those calls are exercised. @@ -204,13 +204,8 @@ test('call* methods POST to /v1/ with the upstream model id and Bearer }, async () => { const [providerModel] = await instance.instance.getProvidedModels(testFetcher); - const result = await instance.instance.callOpenAIChatCompletions( - providerModel, - { messages: [{ role: 'user', content: 'hi' }] }, - undefined, - noopUpstreamCallOptions(), - ); - assertEquals(result.modelKey, 'gpt-oss:120b'); + const result = await collectChatProviderPipeline(instance, 'openaiChatCompletions', providerModel, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); + assertEquals(result.facts['response.provider.modelKey'], 'gpt-oss:120b'); }, ); @@ -250,8 +245,8 @@ test('Anthropic Messages methods serialize typed anthropic-beta metadata only on async () => { const [model] = await instance.instance.getProvidedModels(testFetcher); const opts = noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: ['context-1m', 'advanced-tool-use'] }); - await instance.instance.callAnthropicMessages(model, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); - await instance.instance.callAnthropicMessagesCountTokens(model, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); + await collectChatProviderPipeline(instance, 'anthropicMessages', model, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); + await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', model, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); }, ); diff --git a/packages/provider-ollama/__tests__/usage-probe_test.ts b/packages/provider-ollama/__tests__/usage-probe_test.ts index 0e7b821128..abc661b3a5 100644 --- a/packages/provider-ollama/__tests__/usage-probe_test.ts +++ b/packages/provider-ollama/__tests__/usage-probe_test.ts @@ -10,7 +10,7 @@ import { refreshOllamaUsageProbe, } from '../src/usage-probe.ts'; import { directFetcher, initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; -import { assertEquals, assertRejects, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, assertEquals, assertRejects, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; const UPSTREAM_ID = 'up_ollama_usage'; @@ -128,12 +128,7 @@ const callChat = async (record: UpstreamRecord, onUsageProbe: () => void): Promi return new Response('data: [DONE]\n\n', { status: 200, headers: { 'content-type': 'text/event-stream' } }); }, async () => { - await provider.instance.callOpenAIChatCompletions( - stubProviderModel({ providerData: 'gpt-oss:120b' }), - { messages: [] }, - undefined, - noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } }), - ); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', stubProviderModel({ providerData: 'gpt-oss:120b' }), { messages: [] }, undefined, noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } })); await Promise.all(pending); }, ); @@ -165,12 +160,7 @@ test('token counting leaves the account windows untouched and arms no probe', as return new Response('{"input_tokens":1}', { status: 200 }); }, async () => { - await provider.instance.callAnthropicMessagesCountTokens( - stubProviderModel({ providerData: 'gpt-oss:120b' }), - { messages: [], max_tokens: 16 }, - undefined, - { ...noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } }), anthropicBeta: [] }, - ); + await collectChatProviderPipeline(provider, 'anthropicMessagesCountTokens', stubProviderModel({ providerData: 'gpt-oss:120b' }), { messages: [], max_tokens: 16 }, undefined, { ...noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } }), anthropicBeta: [] }); await Promise.all(pending); }, ); diff --git a/packages/provider-ollama/src/pipelines.ts b/packages/provider-ollama/src/pipelines.ts index b64a040aa0..d4583a2897 100644 --- a/packages/provider-ollama/src/pipelines.ts +++ b/packages/provider-ollama/src/pipelines.ts @@ -4,15 +4,15 @@ import { prepareOllamaRequest } from './stages/prepare-request.ts'; import type { OllamaUpstreamState } from './state.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import { decodeProviderResponse, observeProviderCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction } from '@floway-dev/provider'; import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createOllamaPipelines = (upstreamId: string, config: OllamaUpstreamConfig, state: OllamaUpstreamState): ProviderPipelines => { const observe = observeOllamaAccount(upstreamId, config, state); return { openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('ollama.openaiChatCompletions', [observe, decodeProviderResponse('openaiChatCompletions'), prepareOllamaRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), - openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('ollama.openaiResponses', [observe, selectProviderResponsesAction('generate'), decodeProviderResponse('openaiResponses'), prepareOllamaRequest(config, 'openaiResponses'), observeProviderCall, http]), - openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('ollama.openaiResponsesCompact', [observe, selectProviderResponsesAction('compact'), decodeProviderResponse('openaiResponsesCompact'), prepareOllamaRequest(config, 'openaiResponsesCompact'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('ollama.openaiResponses', [observe, selectProviderResponsesAction('generate'), decodeProviderResponse('openaiResponses'), prepareOllamaRequest(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('ollama.openaiResponsesCompact', [observe, selectProviderResponsesAction('compact'), decodeProviderResponse('openaiResponsesCompact'), prepareOllamaRequest(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('ollama.anthropicMessages', [observe, decodeProviderResponse('anthropicMessages'), prepareOllamaRequest(config, 'anthropicMessages'), observeProviderCall, http]), anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('ollama.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareOllamaRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('ollama.openaiCompletions', [observe, prepareOllamaRequest(config, 'openaiCompletions'), observeProviderCall, http]), diff --git a/packages/provider-ollama/src/stages/prepare-request.ts b/packages/provider-ollama/src/stages/prepare-request.ts index 3aea95a1cd..d92178508e 100644 --- a/packages/provider-ollama/src/stages/prepare-request.ts +++ b/packages/provider-ollama/src/stages/prepare-request.ts @@ -46,7 +46,7 @@ export const prepareOllamaRequest = (config: OllamaUp : { ...payload, ...(operation === 'openaiChatCompletions' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' || operation === 'anthropicMessages' ? { stream: true } : {}), model: modelKey }; const base = config.apiKey ? [['Authorization', secret(`Bearer ${config.apiKey}`)] as const] : []; let headers = mergeHttpHeaders(withHttpContentType(base, body, encoding), facts['request.http.headers']); - if ('request.provider.anthropicBeta' in facts) { + if (operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens') { const beta = facts['request.provider.anthropicBeta'] as readonly string[]; headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index e6eb0ddad8..641e195103 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -11,7 +11,7 @@ export type { AnthropicMessagesCountTokensBody, ChatProviderOperation, NonChatPr export { providerModelFacts } from './pipeline.ts'; export { decodeProviderResponse } from './stages/decode-response.ts'; export { selectProviderResponsesAction } from './stages/select-responses-action.ts'; -export { observeProviderCall } from './stages/observe-call.ts'; +export { observeProviderCall, observeProviderResponsesCall } from './stages/observe-call.ts'; export { mergeHttpHeaders, replaceHttpHeader, withHttpContentType } from './http-headers.ts'; export type { InternalDebugError } from './error.ts'; diff --git a/packages/provider/src/stages/decode-response.ts b/packages/provider/src/stages/decode-response.ts index 461833af83..f33f8b397e 100644 --- a/packages/provider/src/stages/decode-response.ts +++ b/packages/provider/src/stages/decode-response.ts @@ -6,7 +6,8 @@ import { isEventStreamMediaType } from '@floway-dev/protocols/common'; import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; import { parseOpenAIResponsesStream } from '@floway-dev/protocols/openai-responses'; -type Request = Pick & { 'request.provider.responsesAction': 'generate' | 'compact' }; +type Request = Pick; +type Up = HttpResponseFacts & { 'response.provider.responsesAction': 'generate' | 'compact' }; type Answer = HttpResponseFacts & Pick, 'response.provider.output'>; const parsedBody = (text: string): unknown => { @@ -23,18 +24,18 @@ const protocolFailure = (exchange: HttpResponseExchange, text: string): Provider }; }; -export const decodeProviderResponse = (operation: O) => defineStage, ProviderChatServices>({ +export const decodeProviderResponse = (operation: O) => defineStage, ProviderChatServices>({ name: `decodeProvider${operation.replace(/^openai/, 'OpenAI').replace(/^anthropic/, 'Anthropic')}`, through: { - request: { needs: ['request.http.callId', ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : [])], consumes: [], provides: [] }, - response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.body'], provides: ['response.http.body', 'response.provider.output'] }, + request: { needs: ['request.http.callId'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body', ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['response.provider.responsesAction' as const] : [])], consumes: ['response.http.body'], provides: ['response.http.body', 'response.provider.output'] }, }, execute: async (facts, next, use) => { const back = await next(move({ ...facts })); const exchange = back['response.http.exchange']; const output = (answer: ProviderOperationOutputs[O] | ProviderProtocolFailure | null) => move({ ...back, 'response.provider.output': answer }); if (exchange.type === 'transportFailure' || exchange.status < 200 || exchange.status >= 300) return output(null); - if ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact' || operation === 'anthropicMessagesCountTokens') { + if ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && back['response.provider.responsesAction'] === 'compact' || operation === 'anthropicMessagesCountTokens') { const text = await exchangeResponse(exchange).text(); if (exchange.body !== null) setRelease(exchange.body, async () => {}); let body: unknown; diff --git a/packages/provider/src/stages/observe-call.ts b/packages/provider/src/stages/observe-call.ts index d2f970a5a9..100cc6ef69 100644 --- a/packages/provider/src/stages/observe-call.ts +++ b/packages/provider/src/stages/observe-call.ts @@ -2,13 +2,14 @@ import type { ProviderResponse, ProviderServices } from '../pipeline.ts'; import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; import { defineStage, move } from '@floway-dev/pipeline'; -type Request = { 'request.provider.modelKey': string }; +type Request = { 'request.provider.modelKey': string; 'request.provider.responsesAction': 'generate' | 'compact' }; +type Response = ProviderResponse & { 'response.provider.responsesAction'?: 'generate' | 'compact' }; -export const observeProviderCall = defineStage({ - name: 'observeProviderCall', +const observeCall = (responses: boolean) => defineStage({ + name: responses ? 'observeProviderResponsesCall' : 'observeProviderCall', through: { - request: { needs: ['request.provider.modelKey'], consumes: [], provides: [] }, - response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + request: { needs: ['request.provider.modelKey', ...(responses ? ['request.provider.responsesAction' as const] : [])], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', ...(responses ? ['response.provider.responsesAction' as const] : [])] }, }, execute: async (facts, next) => { const back = await next(move({ ...facts })); @@ -17,6 +18,10 @@ export const observeProviderCall = defineStage = Extract extends { frames: AsyncIterable } ? F : never; + +export const collectChatProviderPipeline = async ( + provider: Provider, + operation: O, + model: ProviderModel, + payload: ProviderOperationPayloads[O], + signal: AbortSignal | undefined = undefined, + options: UpstreamCallOptions | AnthropicMessagesUpstreamCallOptions = noopUpstreamCallOptions(), +): Promise<{ facts: ProviderOperationResponse & Record; output: ProviderChatResponse['response.provider.output']; frames: Frame[]; response: Response | null }> => { + const pipeline = provider.pipelines[operation]; + if (pipeline === undefined) throw new Error(`${provider.kind} has no ${operation} pipeline`); + const entry = move({ + 'request.provider.model': providerModelFacts(model), + 'request.provider.payload': payload, + 'request.http.callId': 0, + 'request.http.headers': [...options.headers], + ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? { 'request.provider.anthropicBeta': (options as AnthropicMessagesUpstreamCallOptions).anthropicBeta } : {}), + }) as Handed>; + const services: ProviderChatServices = { httpCall: () => ({ ...options, signal }), recordProtocolFrames: frames => frames }; + const executed = await run, ProviderOperationResponse, ProviderChatServices>(pipeline, entry, services); + const facts = executed.facts as ProviderOperationResponse & Record; + const exchange = facts['response.http.exchange']; + const output = facts['response.provider.output'] as ProviderChatResponse['response.provider.output']; + const frames: Frame[] = []; + let response: Response | null = null; + try { + if (exchange.type === 'transportFailure') throw exchange.error; + if (output !== null && 'kind' in output && output.kind === 'stream') { + for await (const frame of output.frames) frames.push(frame as Frame); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + } else if (output === null) { + const bytes = await exchangeResponse(exchange).arrayBuffer(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + response = new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]) }); + } + } catch (error) { + try { await executed.drain(); } catch (cleanupError) { + if (cleanupError !== error) throw new AggregateError([error, cleanupError], 'Provider test call and cleanup failed', { cause: error }); + } + throw error; + } + await executed.drain(); + return { facts, output, frames, response }; +}; diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index a88fcf7e28..2daf3ceab5 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -1,3 +1,6 @@ +export { applyProviderStage, type ProviderStageProbe, type OpenAIChatCompletionsProbe, type OpenAIResponsesProbe, type AnthropicMessagesProbe } from './provider-stage.ts'; +export { stubChatProviderPipeline, type StubChatProviderCall } from './stub-chat-provider-pipeline.ts'; +export { collectChatProviderPipeline } from './chat-provider-pipeline.ts'; export { assert, assertEquals, diff --git a/packages/test-utils/src/provider-stage.ts b/packages/test-utils/src/provider-stage.ts new file mode 100644 index 0000000000..5dd42f5ff6 --- /dev/null +++ b/packages/test-utils/src/provider-stage.ts @@ -0,0 +1,67 @@ +import { takeHttpResponse, type HttpExchange, type HttpHeaders } from '@floway-dev/http/pipeline'; +import { compose, defineStage, isSecret, move, run, type Stage } from '@floway-dev/pipeline'; +import { providerModelFacts, type ProviderModel, type ProviderOperationOutputs, type ProviderOperationPayloads, type ProviderProtocolFailure } from '@floway-dev/provider'; + +type ProtocolFrame = Extract['frames'] extends AsyncIterable + ? F extends { type: 'event'; event: unknown } ? Omit & { event: T } : F + : never; +type OpenAIChatCompletionsPayload = ProviderOperationPayloads['openaiChatCompletions'] & { model: string }; +type CanonicalOpenAIResponsesPayload = ProviderOperationPayloads['openaiResponses'] & { model: string }; +type AnthropicMessagesPayload = ProviderOperationPayloads['anthropicMessages'] & { model: string }; + +export interface ProviderStageProbe

{ + payload: P; + headers: Headers; + model: ProviderModel; + action?: 'generate' | 'compact'; + anthropicBeta?: readonly string[]; +} +export type OpenAIChatCompletionsProbe = ProviderStageProbe; +export type OpenAIResponsesProbe = ProviderStageProbe & { action: 'generate' | 'compact' }; +export type AnthropicMessagesProbe = ProviderStageProbe & { anthropicBeta: readonly string[] }; + +type Output = { kind: 'stream'; frames: AsyncIterable> } | { kind: 'value'; body: unknown } | ProviderProtocolFailure | null; +type Facts = Record & { 'response.http.exchange': HttpExchange; 'response.provider.output': Output }; + +export const applyProviderStage = async

(stage: Stage, probe: ProviderStageProbe

, reply: () => Promise, extraFacts: Readonly> = {}): Promise => { + let template: unknown; + const terminal = defineStage, Facts>({ + name: 'testProtocolReply', + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.modelKey', 'response.provider.output'] }, + execute: async facts => { + probe.payload = facts['request.provider.payload'] as P; + probe.headers = new Headers((facts['request.http.headers'] as HttpHeaders).map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value])); + if ('request.provider.anthropicBeta' in facts) probe.anthropicBeta = facts['request.provider.anthropicBeta'] as readonly string[]; + template = await reply(); + const value = typeof template === 'object' && template !== null ? template as Record : null; + let exchange: HttpExchange = { type: 'response', status: 200, statusText: 'OK', headers: [], body: null }; + let output: Output = null; + if (value !== null) { + if (value.type === 'events' || value.ok === true && value.action !== 'compact') output = { kind: 'stream', frames: value.events as AsyncIterable> }; + else if (value.ok === true && value.action === 'compact') output = { kind: 'value', body: value.result }; + else if (value.response instanceof Response) exchange = takeHttpResponse(value.response); + else if (value.type === 'api-error') exchange = takeHttpResponse(new Response(value.body as Uint8Array, { status: value.status as number, headers: value.headers as Headers })); + } + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.called': true, 'response.provider.previousCalls': [], 'response.provider.modelKey': probe.model.id, 'response.provider.output': output }); + }, + }); + const executed = await run(compose, Facts>('testProviderStage', [stage, terminal]), move({ + 'request.provider.payload': probe.payload, + 'request.provider.model': providerModelFacts(probe.model), + 'request.provider.modelKey': probe.model.id, + 'request.http.headers': [...probe.headers], + 'request.http.callId': 0, + ...(probe.action === undefined ? {} : { 'request.provider.responsesAction': probe.action }), + ...(probe.anthropicBeta === undefined ? {} : { 'request.provider.anthropicBeta': probe.anthropicBeta }), + ...extraFacts, + }), { recordProtocolFrames: >(frames: AsyncIterable): AsyncIterable => frames }); + const output = executed.facts['response.provider.output']; + const value = typeof template === 'object' && template !== null ? template as Record : null; + let result: unknown = template; + if (value !== null && output !== null) { + if ('kind' in output) result = output.kind === 'stream' ? { ...value, events: output.frames } : { ...value, result: output.body }; + else if (value.type === 'api-error') result = { ...value, status: output.status, headers: new Headers({ 'content-type': 'application/json' }), body: new TextEncoder().encode(JSON.stringify(output.body)) }; + } + await executed.drain(); + return result as R; +}; diff --git a/packages/test-utils/src/stub-chat-provider-pipeline.ts b/packages/test-utils/src/stub-chat-provider-pipeline.ts new file mode 100644 index 0000000000..ccb469186c --- /dev/null +++ b/packages/test-utils/src/stub-chat-provider-pipeline.ts @@ -0,0 +1,84 @@ +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { compose, defineStage, isSecret, move, type Handed } from '@floway-dev/pipeline'; +import type { AnthropicMessagesUpstreamCallOptions, ChatProviderOperation, ProviderChatServices, ProviderModel, ProviderOperationOutputs, ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipeline, ProviderResponse, UpstreamCallOptions } from '@floway-dev/provider'; +import { decodeProviderResponse } from '@floway-dev/provider'; + +type Frame = Extract extends { frames: AsyncIterable } ? F : never; +type TransportFailure = { type: 'transportFailure'; error: unknown }; +type Reply = ResponsesReply | StreamReply<'openaiChatCompletions'> | StreamReply<'anthropicMessages'> | CountReply | TransportFailure; +type AnyFrame = Frame<'openaiChatCompletions'> | Frame<'openaiResponses'> | Frame<'anthropicMessages'>; +type StreamReply = { ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } | { ok: false; response: Response; modelKey: string }; +type ResponsesReply = (StreamReply<'openaiResponses'> & { action: 'generate' }) | { action: 'compact'; ok: true; result: Extract['body']; modelKey: string } | { action: 'compact'; ok: false; response: Response; modelKey: string }; +type CountReply = { response: Response; modelKey: string }; +export type StubChatProviderCall = + O extends 'openaiResponses' | 'openaiResponsesCompact' + ? (model: ProviderModel, payload: ProviderOperationPayloads[O], action: 'generate' | 'compact', signal: AbortSignal | undefined, options: UpstreamCallOptions) => Promise + : O extends 'anthropicMessagesCountTokens' + ? (model: ProviderModel, payload: ProviderOperationPayloads[O], signal: AbortSignal | undefined, options: AnthropicMessagesUpstreamCallOptions) => Promise + : O extends 'anthropicMessages' + ? (model: ProviderModel, payload: ProviderOperationPayloads[O], signal: AbortSignal | undefined, options: AnthropicMessagesUpstreamCallOptions) => Promise | TransportFailure> + : (model: ProviderModel, payload: ProviderOperationPayloads[O], signal: AbortSignal | undefined, options: UpstreamCallOptions) => Promise | TransportFailure>; + +const responseFromFrames = (operation: ChatProviderOperation, frames: AsyncIterable, sourceHeaders: Headers | undefined): Response => { + const iterator = frames[Symbol.asyncIterator](); + const encoder = new TextEncoder(); + const body = new ReadableStream({ + pull: async controller => { + try { + for (;;) { + const item = await iterator.next(); + if (item.done) { controller.close(); return; } + const frame = item.value; + if (frame.type === 'done') { + if (operation === 'anthropicMessages') continue; + controller.enqueue(encoder.encode('data: [DONE]\n\n')); + return; + } + const eventName = operation === 'openaiChatCompletions' ? '' : `event: ${(frame.event as { type: string }).type}\n`; + controller.enqueue(encoder.encode(`${eventName}data: ${JSON.stringify(frame.event)}\n\n`)); + return; + } + } catch (error) { controller.error(error); } + }, + cancel: async () => { await iterator.return?.(); }, + }); + const headers = new Headers(sourceHeaders); + headers.set('content-type', 'text/event-stream'); + return new Response(body, { headers }); +}; + +export const stubChatProviderPipeline = (operation: O, call: StubChatProviderCall): ProviderPipeline => { + type Entry = ProviderOperationRequest & { 'request.provider.anthropicBeta': readonly string[] }; + type Ready = Entry & ProviderResponse & { 'request.provider.responsesAction': 'generate' | 'compact'; 'response.provider.responsesAction': 'generate' | 'compact' }; + const invoke = defineStage, ProviderOperationResponse, ProviderChatServices>({ + name: `stubProviderInvoke${operation}`, + through: { + request: { needs: [...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], consumes: [], provides: ['request.provider.responsesAction', 'response.provider.responsesAction', 'response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.modelKey'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => { + const modelFacts = facts['request.provider.model']; + const model: ProviderModel = { ...modelFacts, enabledFlags: new Set(modelFacts.enabledFlags) }; + const httpCall = use.httpCall(facts['request.http.callId']); + const options: AnthropicMessagesUpstreamCallOptions = { ...httpCall, headers: new Headers(facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value])), anthropicBeta: facts['request.provider.anthropicBeta'] }; + let reply: Reply; + if (operation === 'openaiResponses' || operation === 'openaiResponsesCompact') { + reply = await httpCall.wrapUpstreamCall(() => (call as StubChatProviderCall<'openaiResponses'>)(model, facts['request.provider.payload'] as ProviderOperationPayloads['openaiResponses'], operation === 'openaiResponsesCompact' ? 'compact' : 'generate', httpCall.signal, options)); + } else { + reply = await httpCall.wrapUpstreamCall(() => (call as unknown as (model: ProviderModel, payload: unknown, signal: AbortSignal | undefined, options: AnthropicMessagesUpstreamCallOptions) => Promise)(model, facts['request.provider.payload'] as ProviderOperationPayloads['anthropicMessages'], httpCall.signal, options)); + } + if ('type' in reply) return move({ ...await next(move({ ...facts, 'request.provider.responsesAction': operation === 'openaiResponsesCompact' ? 'compact' : 'generate', 'response.provider.responsesAction': operation === 'openaiResponsesCompact' ? 'compact' : 'generate', 'response.http.exchange': reply, 'response.http.body': null, 'response.provider.called': false, 'response.provider.previousCalls': [], 'response.provider.modelKey': model.id }) as Ready & Record) }) as Handed>; + const response = 'response' in reply ? reply.response : 'events' in reply + ? responseFromFrames(operation, reply.events, reply.headers) + : Response.json(reply.result); + const exchange = takeHttpResponse(response); + return move({ ...await next(move({ ...facts, 'request.provider.responsesAction': 'action' in reply ? reply.action : operation === 'openaiResponsesCompact' ? 'compact' : 'generate', 'response.provider.responsesAction': 'action' in reply ? reply.action : operation === 'openaiResponsesCompact' ? 'compact' : 'generate', 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.called': true, 'response.provider.previousCalls': [], 'response.provider.modelKey': reply.modelKey }) as Ready & Record) }) as Handed>; + }, + }); + const reply = defineStage({ + name: 'stubProviderReply', + return: { provides: ['response.provider.responsesAction', 'response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.modelKey'] }, + execute: async facts => move({ ...facts }) as Ready & Record, + }); + return compose, ProviderOperationResponse>(`stub.${operation}`, [invoke, decodeProviderResponse(operation), reply]); +}; From 977e87845a711c2d722b5e5d94b1b7b1eb837184 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:31:52 +0800 Subject: [PATCH 125/156] fix: settle chat usage when quantity formatting fails Finish each native meter once and reject its produced usage with the original formatting exception. Read failures still retain the measured partial quantities and mark the stream failed. Drive the actual Custom HTTP chain with an invalid numeric usage value to verify the client protocol error, immediate Deferred rejection, known called identity, and original teardown error chain. Complete gateway fixture migration away from removed provider instance methods. All 73 chat test files and 1120 cases pass; scoped lint passes. --- .../chat/anthropic-messages/http_test.ts | 7 +++-- .../chat/gemini-generate-content/http_test.ts | 7 +++-- .../chat/openai-chat-completions/http_test.ts | 5 +-- .../affinity/copilot-roundtrip_test.ts | 18 +++++------ .../chat/openai-responses/http_test.ts | 15 ++++----- .../data-plane/chat/provider-handoffs_test.ts | 31 +++++++++++++++++-- .../web-search/alpha-search/upstream_test.ts | 7 +---- .../test-utils/chat-provider-pipelines.ts | 4 +-- .../chat/anthropic-messages/meter.ts | 19 ++++++++---- .../chat/openai-chat-completions/meter.ts | 19 ++++++++---- .../data-plane/chat/openai-responses/meter.ts | 19 ++++++++---- 11 files changed, 97 insertions(+), 54 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index bc14af8426..e9b60eb08a 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -109,14 +109,15 @@ const makeCandidate = (overrides: { callAnthropicMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; } = {}): ModelCandidate => { const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ + const calls = { callAnthropicMessages: overrides.callAnthropicMessages, callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, - }); + }; + const provider = stubProvider(); return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(provider), instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(calls), instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts index f5d0245067..049b7d6eb6 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts @@ -111,15 +111,16 @@ const makeCandidate = (overrides: { : targetApi === 'anthropicMessages' ? { anthropicMessages: {} } : { openaiResponses: {} }); - const provider = stubProvider({ + const calls = { callOpenAIChatCompletions: overrides.callOpenAIChatCompletions, callAnthropicMessages: overrides.callAnthropicMessages, callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, - }); + }; + const provider = stubProvider(); return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(provider), instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(calls), instance: provider, }, model: stubInternalModel({ endpoints }, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts index 07a0f470d8..a4b9898105 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts @@ -109,11 +109,12 @@ const makeCandidate = (overrides: { callOpenAIChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; } = {}): ModelCandidate => { const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ callOpenAIChatCompletions: overrides.callOpenAIChatCompletions }); + const calls = { callOpenAIChatCompletions: overrides.callOpenAIChatCompletions }; + const provider = stubProvider(); return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(provider), instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(calls), instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts index 5470fb6c1c..1f96e75b16 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts @@ -8,7 +8,7 @@ import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { initProviderRepo, providerModelOf, type UpstreamRecord } from '@floway-dev/provider'; import { clearInProcessCopilotTokenCache, copilotProviderModule } from '@floway-dev/provider-copilot'; -import { noopUpstreamCallOptions, sseResponse, stubModelCandidate, stubProvider, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, noopUpstreamCallOptions, sseResponse, stubModelCandidate, stubProvider, withMockedFetch } from '@floway-dev/test-utils'; const upstream: UpstreamRecord = { id: 'up-copilot', @@ -117,17 +117,16 @@ test('Copilot item-id and generic affinity trailers compose and unwrap in bounda return sseResponse(); }, async () => { - const first = await provider.instance.callOpenAIResponses( - providerModelOf(candidate), + const first = await collectChatProviderPipeline( + provider, 'openaiResponses', providerModelOf(candidate), { input: [], stream: true, store: false }, - 'generate', undefined, noopUpstreamCallOptions(), ); - if (!first.ok || first.action !== 'generate') throw new Error('expected first Copilot stream'); + if (first.output === null || !('kind' in first.output) || first.output.kind !== 'stream') throw new Error('expected first Copilot stream'); const codec = new AffinityCodec('00'.repeat(32)); - const publicEvents = await collectEvents(wrapOpenAIResponsesAffinityEgress(first.events, { + const publicEvents = await collectEvents(wrapOpenAIResponsesAffinityEgress({ async *[Symbol.asyncIterator]() { yield* first.frames; } }, { codec, affinity: { upstreamId: provider.upstreamId, @@ -155,14 +154,13 @@ test('Copilot item-id and generic affinity trailers compose and unwrap in bounda expect(foreign.input[0]).toEqual({ type: 'reasoning', id: publicItem.id, summary: [] }); const { model: _model, ...exactBody } = exact; - const second = await provider.instance.callOpenAIResponses( - providerModelOf(candidate), + const second = await collectChatProviderPipeline( + provider, 'openaiResponses', providerModelOf(candidate), exactBody, - 'generate', undefined, noopUpstreamCallOptions(), ); - if (!second.ok || second.action !== 'generate') throw new Error('expected replay Copilot stream'); + if (second.output === null || !('kind' in second.output) || second.output.kind !== 'stream') throw new Error('expected replay Copilot stream'); }, ); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts index 0541b2d3dc..3190957413 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts @@ -7,7 +7,7 @@ import type { AuthVars } from '../../../../src/middleware/auth.ts'; import { initRepo } from '../../../../src/repo/index.ts'; import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; -import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; +import { stubChatProviderPipelines, type ChatFixtureCalls } from '../../../test-utils/chat-provider-pipelines.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; @@ -130,9 +130,10 @@ const makeCandidate = (overrides: { } = {}): ModelCandidate => { const upstream = overrides.upstream ?? 'up_test'; const endpoints = overrides.endpoints ?? { openaiChatCompletions: {}, openaiResponses: {}, anthropicMessages: {} }; - const provider = stubProvider({ + const calls = { callOpenAIResponses: overrides.callOpenAIResponses, - }); + }; + const provider = stubProvider(); return { provider: { upstreamId: upstream, @@ -141,7 +142,7 @@ const makeCandidate = (overrides: { inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, pipelines: stubChatProviderPipelines(provider), + modelsCache: null, pipelines: stubChatProviderPipelines(calls), instance: provider, }, model: stubInternalModel({ @@ -804,7 +805,7 @@ const translatedCustomCandidate = ( callExec = false, ): ModelCandidate => { const candidate = makeCandidate({ upstream: `up_${target}`, endpoints: { [target]: {} } }); - const instance = stubProvider({ + const calls: ChatFixtureCalls = { callOpenAIChatCompletions: async (_model, body) => { observe(body as unknown as Record); const chunk = (choices: OpenAIChatCompletionsStreamEvent['choices']): OpenAIChatCompletionsStreamEvent => ({ id: 'chat_exec', object: 'chat.completion.chunk', created: 0, model: 'test-model', choices }); @@ -830,8 +831,8 @@ const translatedCustomCandidate = ( })(), }; }, - }); - return { ...candidate, provider: { ...candidate.provider, instance, pipelines: stubChatProviderPipelines(instance) } }; + }; + return { ...candidate, provider: { ...candidate.provider, pipelines: stubChatProviderPipelines(calls) } }; }; for (const target of ['openaiChatCompletions', 'anthropicMessages'] as const) { diff --git a/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts b/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts index 74acc8c95e..f3370ea5ac 100644 --- a/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts @@ -16,7 +16,7 @@ const config = { baseUrl: 'https://custom.example', authStyle: 'none' as const, const first = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'wire-model', choices: [{ index: 0, delta: { content: 'partial' }, finish_reason: null }] }; const usage = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'wire-model', choices: [], usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } }; -for (const client of ['unread', 'partial', 'complete'] as const) test(`typed provider handoff preserves HTTP 201 and drains one reading after ${client} client output`, async () => { +const runWire = async (response: Response) => { const gateway = mockChatGatewayCtx({ wantsStream: true }); const base = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }); const plain = stubModelCandidate({ model: { id: 'alias', endpoints: { openaiChatCompletions: {} } }, providerData: 'wire-model' }); @@ -29,7 +29,7 @@ for (const client of ['unread', 'partial', 'complete'] as const) test(`typed pro if (!isReplayableBody(init.body)) throw new Error('JSON HTTP content must be replayable'); calls += 1; expect(await new Response(init.body.open()).json()).toMatchObject({ model: 'wire-model', stream: true, messages: [{ role: 'user', content: 'hello' }], stream_options: { include_usage: true } }); - return new Response([first, usage].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { status: 201, headers: { 'content-type': 'text/event-stream', 'x-upstream': 'kept' } }); + return response; }, }; const [selector] = base.services.rememberCandidates([candidate]); @@ -47,6 +47,11 @@ for (const client of ['unread', 'partial', 'complete'] as const) test(`typed pro selectAffinity: () => {}, dump: (event: Event) => { events.push(event); }, }); + return { executed, events, calls: () => calls }; +}; + +for (const client of ['unread', 'partial', 'complete'] as const) test(`typed provider handoff preserves HTTP 201 and drains one reading after ${client} client output`, async () => { + const { executed, events, calls } = await runWire(new Response([first, usage].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { status: 201, headers: { 'content-type': 'text/event-stream', 'x-upstream': 'kept' } })); expect(executed.facts['response.http.status']).toBe(201); expect(executed.facts['response.http.headers']).toContainEqual(['x-upstream', 'kept']); const frames = executed.facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable; @@ -64,7 +69,27 @@ for (const client of ['unread', 'partial', 'complete'] as const) test(`typed pro expect(outcome.failed).toBe(client !== 'complete'); expect(outcome.billable).toHaveLength(1); expect(outcome.billable[0].quantities).toMatchObject({ input_tokens: '10', output_tokens: '5' }); - expect(calls).toBe(1); + expect(calls()).toBe(1); expect(events).toContainEqual(expect.objectContaining({ type: 'stage.entered', name: 'http' })); expect(events).toContainEqual(expect.objectContaining({ type: 'stage.entered', facts: expect.objectContaining({ 'request.provider.payload': expect.objectContaining({ messages: [{ role: 'user', content: 'hello' }] }), 'request.provider.model': expect.objectContaining({ enabledFlags: [] }) }) })); }); + +test('an actual upstream quantity-formatting fault rejects produced usage with the original error', async () => { + const raw = `data: ${JSON.stringify(first)}\n\ndata: {"id":"c1","object":"chat.completion.chunk","model":"wire-model","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":1e999,"total_tokens":1}}\n\ndata: [DONE]\n\n`; + const { executed, calls } = await runWire(new Response(raw, { headers: { 'content-type': 'text/event-stream' } })); + const sent: SseFrame[] = []; + for await (const frame of executed.facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) sent.push(frame); + const reading = executed.facts['response.chat.openaiChatCompletions.streamedUsage'] as Deferred; + let original: unknown; + try { await reading; } catch (error) { original = error; } + expect(original).toBeInstanceOf(TypeError); + expect((original as Error).message).toContain('Infinity'); + expect(JSON.parse(sent.at(-1)!.data)).toMatchObject({ error: { name: 'TypeError', message: (original as Error).message } }); + expect(executed.facts['response.usage.billable']).toMatchObject([{ quantities: {} }]); + expect(calls()).toBe(1); + let teardown: unknown; + try { await executed.drain(); } catch (error) { teardown = error; } + const causes = teardown instanceof AggregateError ? teardown.errors as unknown[] : [teardown]; + expect(causes.length).toBeGreaterThan(0); + expect(causes.every(error => error === original)).toBe(true); +}); diff --git a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts b/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts index bc323639f7..feb3b112e5 100644 --- a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts +++ b/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts @@ -36,12 +36,7 @@ const dispatcherFor = async (kind: 'codex' | 'custom', inboundHeaderAllowlist: r return { response: new Response('{}'), modelKey: 'search-model' }; }), }, - instance: stubProvider({ - callAlphaSearch: async (_model, _body, _signal, opts) => { - observedHeaders = opts.headers; - return { response: new Response('{}'), modelKey: 'search-model' }; - }, - }), + instance: stubProvider(), }; resolvedCandidate = stubModelCandidate({ provider }); const dispatcher = await resolveAlphaSearchCandidate({ diff --git a/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts b/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts index d9cbf35e7d..14a957b148 100644 --- a/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts +++ b/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts @@ -2,14 +2,14 @@ import type { GatewayCtx } from '../../src/data-plane/shared/gateway-ctx.ts'; import type { ProviderPipelines, ProviderChatServices } from '@floway-dev/provider'; import { noopUpstreamCallOptions, stubChatProviderPipeline, type StubChatProviderCall } from '@floway-dev/test-utils'; -type Calls = Partial<{ +export type ChatFixtureCalls = Partial<{ callOpenAIChatCompletions: StubChatProviderCall<'openaiChatCompletions'>; callOpenAIResponses: StubChatProviderCall<'openaiResponses'>; callAnthropicMessages: StubChatProviderCall<'anthropicMessages'>; callAnthropicMessagesCountTokens: StubChatProviderCall<'anthropicMessagesCountTokens'>; }>; -export const stubChatProviderPipelines = (calls: Calls): ProviderPipelines => ({ +export const stubChatProviderPipelines = (calls: ChatFixtureCalls): ProviderPipelines => ({ ...(calls.callOpenAIChatCompletions === undefined ? {} : { openaiChatCompletions: stubChatProviderPipeline('openaiChatCompletions', calls.callOpenAIChatCompletions) }), ...(calls.callOpenAIResponses === undefined ? {} : { openaiResponses: stubChatProviderPipeline('openaiResponses', calls.callOpenAIResponses), diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts index 601322ac69..ceb42d4ffe 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts @@ -25,10 +25,9 @@ const meterAnthropicMessages = ( identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, ): { readonly frames: AsyncIterable>; readonly outcome: Deferred } => { - let settle!: (outcome: StreamOutcome) => void; - // Declared as this run's own unfinished work, so the runner waits for it at teardown where - // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); + const completion = Promise.withResolvers(); + const outcome = defer(completion.promise); + let settled = false; // Running out without the terminal frame is what "it did not finish" means, and it is known // at the same moment the usage is. let sawTerminal = false; @@ -36,6 +35,14 @@ const meterAnthropicMessages = ( const readBillableUsage = createAnthropicMessagesBillableUsageReader(); const generator = (async function* () { let reported: BillableUsage | undefined; + const finish = (): void => { + if (settled) return; + settled = true; + try { completion.resolve({ billable: [billedEntity(reported, identity)], failed: failed || !sawTerminal }); } catch (error) { + completion.reject(error); + throw error; + } + }; try { for await (const frame of source) { // Time to first token is measured where the token is, which is the only place that @@ -50,7 +57,7 @@ const meterAnthropicMessages = ( if (isAnthropicMessagesTerminalFrame(frame)) { sawTerminal = true; failed = frame.type === 'event' && frame.event.type === 'error'; - settle({ billable: [billedEntity(reported, identity)], failed }); + finish(); } yield frame; // The turn is over, so there is nothing further to read. An upstream that holds the @@ -65,7 +72,7 @@ const meterAnthropicMessages = ( // Reached however the frames ended — the terminal event, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle({ billable: [billedEntity(reported, identity)], failed: failed || !sawTerminal }); + finish(); } })(); return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts index f6a04f2bb0..732905f28f 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts @@ -24,15 +24,22 @@ const meterOpenAIChatCompletions = ( identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, ): { readonly frames: AsyncIterable>; readonly outcome: Deferred } => { - let settle!: (outcome: StreamOutcome) => void; - // Declared as this run's own unfinished work, so the runner waits for it at teardown where - // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); + const completion = Promise.withResolvers(); + const outcome = defer(completion.promise); + let settled = false; // A clean EOF completes Chat Completions even when no [DONE] frame was sent. let completed = false; let failed = false; const generator = (async function* () { let reported: BillableUsage | undefined; + const finish = (): void => { + if (settled) return; + settled = true; + try { completion.resolve({ billable: [billedEntity(reported, identity)], failed: failed || !completed }); } catch (error) { + completion.reject(error); + throw error; + } + }; try { for await (const frame of source) { // Time to first token is measured where the token is, which is the only place that @@ -47,7 +54,7 @@ const meterOpenAIChatCompletions = ( if (isTerminal(frame)) { completed = true; failed = frame.type === 'event' && 'error' in frame.event; - settle({ billable: [billedEntity(reported, identity)], failed }); + finish(); } yield frame; // The terminator is written out before the read stops, because it is what the client @@ -59,7 +66,7 @@ const meterOpenAIChatCompletions = ( // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle({ billable: [billedEntity(reported, identity)], failed: failed || !completed }); + finish(); } })(); return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/meter.ts b/packages/gateway/src/data-plane/chat/openai-responses/meter.ts index b69d336093..9b3b490e6e 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/meter.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/meter.ts @@ -25,16 +25,23 @@ const meterOpenAIResponses = ( identity: TelemetryModelIdentity, attempt: { firstOutputTokenAt: number | null }, ): { readonly frames: AsyncIterable>; readonly outcome: Deferred } => { - let settle!: (outcome: StreamOutcome) => void; - // Declared as this run's own unfinished work, so the runner waits for it at teardown where - // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); + const completion = Promise.withResolvers(); + const outcome = defer(completion.promise); + let settled = false; // Running out without the terminal frame is what "it did not finish" means, and it is known // at the same moment the usage is. let sawTerminal = false; let failed = false; const generator = (async function* () { let reported: BillableUsage | undefined; + const finish = (): void => { + if (settled) return; + settled = true; + try { completion.resolve({ billable: [billedOpenAIResponsesEntity(identity, reported)], failed: failed || !sawTerminal }); } catch (error) { + completion.reject(error); + throw error; + } + }; try { for await (const frame of source) { // Time to first token is measured where the token is, which is the only place that @@ -56,7 +63,7 @@ const meterOpenAIResponses = ( if (isOpenAIResponsesTerminalEvent(frame.event)) { sawTerminal = true; failed = frame.event.type === 'response.failed' || frame.event.type === 'error'; - settle({ billable: [billedOpenAIResponsesEntity(identity, reported)], failed }); + finish(); } yield frame; // The turn is over, so there is nothing further to read. An upstream that holds the @@ -71,7 +78,7 @@ const meterOpenAIResponses = ( // Reached however the frames ended — the terminal event, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle({ billable: [billedOpenAIResponsesEntity(identity, reported)], failed: failed || !sawTerminal }); + finish(); } })(); return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; From 9b8a066d371e9d0d2f1a62fa388061ac9726d215 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:43:50 +0800 Subject: [PATCH 126/156] refactor: declare authored chat response body facts --- .../chat/openai-responses/answer-websocket-warmup.ts | 4 ++-- .../hosted-tools/image-sub-request/prepare.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts b/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts index 50d48958e8..02f4bb41b3 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts @@ -7,8 +7,8 @@ import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult } from '@fl import { providerModelOf } from '@floway-dev/provider'; type Request = Pick; -type Response = Pick; -const provides = ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers'] as const; +type Response = Pick; +const provides = ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.body'] as const; // Codex continues from the response id of its WebSocket prewarm. The source chain has // already staged its input; normal client egress commits that snapshot without inference. diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts index 3f2d98399f..30865297e5 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts @@ -7,9 +7,9 @@ import { openaiImagesRequestWantsStream } from '@floway-dev/protocols/openai-ima export const prepareHostedImageGeneration = defineStage< Fields<'request.imageGeneration.canonical'>, Fields<'request.imageGeneration.canonical' | 'request.openaiImages.canonical' | 'serve.model' | 'ingress.openaiImages.wantsStream'>, - Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, - Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, - Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, GatewayServices >({ name: 'prepareHostedImageGeneration', @@ -17,7 +17,7 @@ export const prepareHostedImageGeneration = defineStage< request: { needs: ['request.imageGeneration.canonical'], consumes: [], provides: ['request.openaiImages.canonical', 'serve.model', 'ingress.openaiImages.wantsStream'] }, response: { needs: [], consumes: [], provides: [] }, }, - return: { provides: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status', 'response.http.headers', 'response.usage.billable'] }, + return: { provides: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable'] }, execute: async (facts, next, use) => { let request; try { From 490e0547c9f51fcaf3ddb0dc84a23f5bd02c402f Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:46:18 +0800 Subject: [PATCH 127/156] refactor: remove legacy provider call surfaces Keep catalog refresh as the transparent provider control callback and expose model operations solely as immutable fact pipelines. Delete duplicate private chains, callable data-plane methods and model-endpoint transport helpers; retain active catalog and account control transports. Move primitive and rule regressions onto actual prepare, HTTP and decoder stages. Remove obsolete FormData/streaming adapters and preserve portable upload metadata. Seven package typechecks and scoped lint pass; 87 HTTP/provider test files and 977 tests pass. --- .../__tests__/stages/audio-request_test.ts | 19 ++ .../fill-empty-namespace-descriptions_test.ts | 8 +- .../prepare-request_test.ts} | 47 ++- packages/provider-azure/package.json | 6 +- packages/provider-azure/src/fetch.ts | 85 ----- .../fill-empty-namespace-descriptions.ts | 21 -- .../interceptors/openai-responses/index.ts | 8 - .../interceptors/openai-responses/types.ts | 15 - packages/provider-azure/src/provider.ts | 96 +----- ...a_test.ts => claude-code-metadata_test.ts} | 2 +- .../{interceptors => }/image-size_test.ts | 2 +- .../anthropic-messages/boundary-chain_test.ts | 65 ---- ...carrier_test.ts => opaque-item-id_test.ts} | 2 +- .../pipeline-headers_test.ts | 31 ++ .../abort-on-tool-argument-whitespace_test.ts | 2 +- .../abort-on-tool-argument-whitespace_test.ts | 2 +- .../openai-responses/item-id-membrane_test.ts | 2 +- packages/provider-copilot/package.json | 6 +- packages/provider-copilot/src/auth.ts | 10 +- ...de-metadata.ts => claude-code-metadata.ts} | 0 packages/provider-copilot/src/control-http.ts | 45 +++ packages/provider-copilot/src/fetch-models.ts | 2 +- packages/provider-copilot/src/fetch.ts | 71 ----- .../{interceptors => }/image-compression.ts | 2 +- .../src/{interceptors => }/image-size.ts | 0 .../apply-top-level-cache-control.ts | 86 ----- .../anthropic-messages/compress-images.ts | 115 ------- .../anthropic-messages/handle-speed-fast.ts | 76 ----- .../interceptors/anthropic-messages/index.ts | 87 ----- .../normalize-anthropic-beta.ts | 35 -- .../promote-thinking-display.ts | 114 ------- .../rewrite-context-window-error.ts | 33 -- .../set-claude-agent-headers.ts | 58 ---- .../anthropic-messages/set-compact-headers.ts | 126 -------- .../set-initiator-header.ts | 40 --- .../set-interaction-id-header.ts | 33 -- .../anthropic-messages/set-vision-header.ts | 40 --- .../strip-cache-control-extensions.ts | 68 ---- .../strip-eager-input-streaming.ts | 25 -- .../interceptors/anthropic-messages/types.ts | 37 --- .../abort-on-tool-argument-whitespace.ts | 69 ---- .../attach-cache-control-markers.ts | 82 ----- .../compress-images.ts | 55 ---- .../openai-chat-completions/index.ts | 23 -- .../set-initiator-header.ts | 42 --- .../set-vision-header.ts | 19 -- .../openai-chat-completions/types.ts | 17 - .../abort-on-tool-argument-whitespace.ts | 69 ---- .../openai-responses/compress-images.ts | 91 ------ .../fill-empty-namespace-descriptions.ts | 22 -- .../openai-responses/force-store-false.ts | 22 -- .../interceptors/openai-responses/index.ts | 34 -- .../openai-responses/item-id-membrane.ts | 279 ---------------- .../openai-responses/set-initiator-header.ts | 38 --- .../openai-responses/set-vision-header.ts | 29 -- .../strip-image-generation.ts | 68 ---- .../openai-responses/strip-service-tier.ts | 27 -- .../interceptors/openai-responses/types.ts | 26 -- .../provider-copilot/src/model-selection.ts | 4 +- .../item-id-carrier.ts => opaque-item-id.ts} | 0 packages/provider-copilot/src/provider.ts | 298 +----------------- .../anthropic-messages/compress-images.ts | 2 +- .../set-claude-agent-headers.ts | 2 +- .../set-interaction-id-header.ts | 2 +- .../abort-on-tool-argument-whitespace.ts | 2 +- .../compress-images.ts | 4 +- .../abort-on-tool-argument-whitespace.ts | 2 +- .../openai-responses/compress-images.ts | 4 +- .../openai-responses/item-id-membrane.ts | 2 +- ...verflow.ts => tool-argument-whitespace.ts} | 2 +- .../__tests__/stages/audio-request_test.ts | 23 ++ .../prepare-request_test.ts} | 69 ++-- packages/provider-custom/src/control-http.ts | 53 ++++ packages/provider-custom/src/fetch-models.ts | 2 +- packages/provider-custom/src/fetch.ts | 87 ----- packages/provider-custom/src/provider.ts | 128 +------- .../__tests__/provider_test.ts | 2 - .../prepare-request_test.ts} | 40 +-- packages/provider-ollama/src/account-probe.ts | 2 +- .../src/{fetch.ts => control-http.ts} | 16 - packages/provider-ollama/src/fetch-models.ts | 2 +- packages/provider-ollama/src/provider.ts | 119 +------ packages/provider-ollama/src/usage-probe.ts | 2 +- packages/provider/__tests__/audio_test.ts | 41 --- packages/provider/__tests__/images_test.ts | 45 ++- .../__tests__/stages/decode-response_test.ts | 24 ++ packages/provider/__tests__/streaming_test.ts | 125 -------- packages/provider/src/audio.ts | 31 -- packages/provider/src/images.ts | 15 - packages/provider/src/index.ts | 5 +- packages/provider/src/provider.ts | 75 +---- packages/provider/src/streaming.ts | 43 --- .../test-utils/src/chat-provider-pipeline.ts | 5 +- packages/test-utils/src/http-pipeline.ts | 20 ++ packages/test-utils/src/index.ts | 2 + .../src/prepared-provider-request.ts | 28 ++ packages/test-utils/src/stubs.ts | 24 +- 97 files changed, 402 insertions(+), 3484 deletions(-) create mode 100644 packages/provider-azure/__tests__/stages/audio-request_test.ts rename packages/provider-azure/__tests__/{interceptors => stages}/openai-responses/fill-empty-namespace-descriptions_test.ts (81%) rename packages/provider-azure/__tests__/{fetch_test.ts => stages/prepare-request_test.ts} (67%) delete mode 100644 packages/provider-azure/src/fetch.ts delete mode 100644 packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts delete mode 100644 packages/provider-azure/src/interceptors/openai-responses/index.ts delete mode 100644 packages/provider-azure/src/interceptors/openai-responses/types.ts rename packages/provider-copilot/__tests__/{interceptors/anthropic-messages/detect-claude-code-metadata_test.ts => claude-code-metadata_test.ts} (94%) rename packages/provider-copilot/__tests__/{interceptors => }/image-size_test.ts (93%) delete mode 100644 packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts rename packages/provider-copilot/__tests__/{interceptors/openai-responses/item-id-carrier_test.ts => opaque-item-id_test.ts} (95%) create mode 100644 packages/provider-copilot/__tests__/stages/anthropic-messages/pipeline-headers_test.ts rename packages/provider-copilot/src/{interceptors/anthropic-messages/detect-claude-code-metadata.ts => claude-code-metadata.ts} (100%) create mode 100644 packages/provider-copilot/src/control-http.ts delete mode 100644 packages/provider-copilot/src/fetch.ts rename packages/provider-copilot/src/{interceptors => }/image-compression.ts (96%) rename packages/provider-copilot/src/{interceptors => }/image-size.ts (100%) delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/index.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts delete mode 100644 packages/provider-copilot/src/interceptors/anthropic-messages/types.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-chat-completions/index.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/index.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts delete mode 100644 packages/provider-copilot/src/interceptors/openai-responses/types.ts rename packages/provider-copilot/src/{interceptors/openai-responses/item-id-carrier.ts => opaque-item-id.ts} (100%) rename packages/provider-copilot/src/{interceptors/shared/whitespace-overflow.ts => tool-argument-whitespace.ts} (92%) create mode 100644 packages/provider-custom/__tests__/stages/audio-request_test.ts rename packages/provider-custom/__tests__/{fetch_test.ts => stages/prepare-request_test.ts} (69%) create mode 100644 packages/provider-custom/src/control-http.ts delete mode 100644 packages/provider-custom/src/fetch.ts rename packages/provider-ollama/__tests__/{fetch_test.ts => stages/prepare-request_test.ts} (50%) rename packages/provider-ollama/src/{fetch.ts => control-http.ts} (61%) delete mode 100644 packages/provider/__tests__/audio_test.ts delete mode 100644 packages/provider/__tests__/streaming_test.ts delete mode 100644 packages/provider/src/streaming.ts create mode 100644 packages/test-utils/src/http-pipeline.ts create mode 100644 packages/test-utils/src/prepared-provider-request.ts diff --git a/packages/provider-azure/__tests__/stages/audio-request_test.ts b/packages/provider-azure/__tests__/stages/audio-request_test.ts new file mode 100644 index 0000000000..87f00935e2 --- /dev/null +++ b/packages/provider-azure/__tests__/stages/audio-request_test.ts @@ -0,0 +1,19 @@ +import { expect, test } from 'vitest'; + +import { prepareAzureRequest } from '../../src/stages/prepare-request.ts'; +import { http } from '@floway-dev/http/pipeline'; +import type { HttpMultipartBody } from '@floway-dev/http/request-content'; +import { compose } from '@floway-dev/pipeline'; +import { observeProviderCall, providerModelFacts, type ProviderRequest, type ProviderResponse } from '@floway-dev/provider'; +import { collectHttpPipeline, noopUpstreamCallOptions, stubProviderModel } from '@floway-dev/test-utils'; + +test('audio prepare omits model fields selected by the deployment URL', async () => { + const file = { bytes: new TextEncoder().encode('audio'), name: 'meeting.wav', type: 'audio/wav' }; + const entries = [{ name: 'model', value: 'public-model' }, { name: 'file', value: file }, { name: 'response_format', value: 'json' }]; + const pipeline = compose, ProviderResponse & Record>('audio.deployment', [prepareAzureRequest({ endpoint: 'https://azure.example', apiKey: 'key', models: [] }, 'openaiAudioTranscriptions'), observeProviderCall, http]); + const executed = await collectHttpPipeline(pipeline, { 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: { upstreamModelId: 'deployment' } })), 'request.provider.payload': { entries }, 'request.http.callId': 0, 'request.http.headers': [] }, { httpCall: () => ({ ...noopUpstreamCallOptions(), fetcher: async () => new Response('{}'), signal: undefined }) }); + const body = executed.facts['request.http.body'] as HttpMultipartBody; + expect(body.entries.map(entry => entry.name)).toEqual(['file', 'response_format']); + expect(body.entries[0]!.value).toBe(file); + expect(executed.facts['request.http.url']).toBe('https://azure.example/openai/deployments/deployment/audio/transcriptions?api-version=2025-04-01-preview'); +}); diff --git a/packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts b/packages/provider-azure/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts similarity index 81% rename from packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts rename to packages/provider-azure/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts index 5a218c453b..c8653731ed 100644 --- a/packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts +++ b/packages/provider-azure/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts @@ -1,11 +1,11 @@ import { test } from 'vitest'; -import { withEmptyNamespaceDescriptionsFilled } from '../../../src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { fillAzureNamespaceDescriptions } from '../../../src/stages/fill-namespace-descriptions.ts'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -36,7 +36,7 @@ test('fills Codex OpenAI Responses Lite namespace descriptions before Azure disp }], }); - await withEmptyNamespaceDescriptionsFilled(ctx, async () => {}); + await applyProviderStage(fillAzureNamespaceDescriptions, ctx, async () => {}); const [item] = ctx.payload.input; if (item?.type !== 'additional_tools') throw new Error('expected additional_tools input'); diff --git a/packages/provider-azure/__tests__/fetch_test.ts b/packages/provider-azure/__tests__/stages/prepare-request_test.ts similarity index 67% rename from packages/provider-azure/__tests__/fetch_test.ts rename to packages/provider-azure/__tests__/stages/prepare-request_test.ts index 0b9a7dfbfa..bbce50ebc2 100644 --- a/packages/provider-azure/__tests__/fetch_test.ts +++ b/packages/provider-azure/__tests__/stages/prepare-request_test.ts @@ -1,19 +1,10 @@ import { test } from 'vitest'; -import { assertAzureUpstreamRecord } from '../src/config.ts'; -import { - azureFetchOpenAIChatCompletions, - azureFetchOpenAIAudioTranscriptions, - azureFetchOpenAIEmbeddings, - azureFetchOpenAIImagesGenerations, - azureFetchAnthropicMessages, - azureFetchAnthropicMessagesCountTokens, - azureFetchOpenAIResponses, - azureFetchOpenAIResponsesCompact, -} from '../src/fetch.ts'; +import { assertAzureUpstreamRecord } from '../../src/config.ts'; +import { prepareAzureRequest } from '../../src/stages/prepare-request.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; -import { directFetcher, identityWrapUpstreamCall } from '@floway-dev/provider'; -import { assertEquals, withMockedFetch } from '@floway-dev/test-utils'; +import { identityWrapUpstreamCall } from '@floway-dev/provider'; +import { collectPreparedProviderRequest, testFetcher, assertEquals, withMockedFetch } from '@floway-dev/test-utils'; const baseRecord: UpstreamRecord = { id: 'up_azure', @@ -57,10 +48,10 @@ test('OpenAI v1 transports apply api-key auth and the canonical paths', async () return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIChatCompletions(config, { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchOpenAIResponses(config, { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchOpenAIResponsesCompact(config, { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchOpenAIEmbeddings(config, { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => ({ upstreamModelId: key }), { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponses'), 'openaiResponses', key => ({ upstreamModelId: key }), { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponsesCompact'), 'openaiResponsesCompact', key => ({ upstreamModelId: key }), { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiEmbeddings'), 'openaiEmbeddings', key => ({ upstreamModelId: key }), { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -81,7 +72,7 @@ test('OpenAI v1 transports apply api-key auth and the canonical paths', async () seen.map(item => item.contentType), ['application/json', 'application/json', 'application/json', 'application/json'], ); - assertEquals(seen[0].body, { model: 'set-by-provider' }); + assertEquals(seen[0].body, { model: 'set-by-provider', stream: true }); }); test('image transports append the Azure preview api-version', async () => { @@ -94,7 +85,7 @@ test('image transports append the Azure preview api-version', async () => { return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIImagesGenerations(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiImagesGenerations'), 'openaiImagesGenerations', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -121,7 +112,7 @@ test('audio transcription selects the deployment on every admitted endpoint shap return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIAudioTranscriptions(config, 'transcribe deployment', { method: 'POST', body: new FormData() }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiAudioTranscriptions'), 'openaiAudioTranscriptions', key => ({ upstreamModelId: key }), { method: 'POST', body: new FormData() }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }, 'transcribe deployment'); }, ); assertEquals(seenUrl, expected); @@ -144,7 +135,7 @@ test('endpoint that already includes /openai/v1 routes through unchanged', async return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponses'), 'openaiResponses', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -173,7 +164,7 @@ test('Foundry project endpoints route OpenAI v1 calls under the project base', a return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponses'), 'openaiResponses', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -202,8 +193,8 @@ test('Foundry project endpoints split OpenAI v1 vs Anthropic surfaces', async () return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponses'), 'openaiResponses', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessages'), 'anthropicMessages', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -235,7 +226,7 @@ test('native Anthropic calls land on the resource Anthropic base when a project return new Response('{}', { status: 200 }); }, async () => { - await azureFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessages'), 'anthropicMessages', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -270,8 +261,8 @@ test('Azure Foundry Anthropic surface uses x-api-key + anthropic-version', async return new Response('{}', { status: 200 }); }, async () => { - await azureFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { extraHeaders: [['anthropic-beta', 'context-1m']], fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchAnthropicMessagesCountTokens(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessages'), 'anthropicMessages', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { extraHeaders: [['anthropic-beta', 'context-1m']], fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessagesCountTokens'), 'anthropicMessagesCountTokens', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -315,7 +306,7 @@ test('Foundry Anthropic messages target URI is accepted and splits per surface', return new Response('{}', { status: 200 }); }, async () => { - await azureFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessages'), 'anthropicMessages', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); diff --git a/packages/provider-azure/package.json b/packages/provider-azure/package.json index 432f72a585..9b92bc8f0b 100644 --- a/packages/provider-azure/package.json +++ b/packages/provider-azure/package.json @@ -4,7 +4,10 @@ "version": "0.0.0", "type": "module", "exports": { - ".": { "import": "./src/index.ts", "types": "./src/index.ts" } + ".": { + "import": "./src/index.ts", + "types": "./src/index.ts" + } }, "scripts": { "typecheck": "tsc --noEmit" @@ -12,7 +15,6 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*" }, diff --git a/packages/provider-azure/src/fetch.ts b/packages/provider-azure/src/fetch.ts deleted file mode 100644 index 9d4572b7ff..0000000000 --- a/packages/provider-azure/src/fetch.ts +++ /dev/null @@ -1,85 +0,0 @@ -import type { AzureUpstreamConfig } from './config.ts'; -import { azureAnthropicBaseUrl, azureOpenAiV1BaseUrl } from './endpoint.ts'; -import { type FetchInit, type UpstreamFetchOptions, joinBaseAndPath } from '@floway-dev/provider'; - -const azureFetchUrl = async ( - config: AzureUpstreamConfig, - surface: 'openai' | 'anthropic', - url: string, - init: FetchInit, - options: UpstreamFetchOptions, -): Promise => { - const headers = new Headers(init.headers); - if (surface === 'anthropic') { - headers.set('x-api-key', config.apiKey); - headers.set('anthropic-version', '2023-06-01'); - } else { - headers.set('api-key', config.apiKey); - } - if (init.body && !headers.has('Content-Type') && !(init.body instanceof FormData)) { - headers.set('Content-Type', 'application/json'); - } - if (options.extraHeaders) { - for (const [key, value] of options.extraHeaders) headers.set(key, value); - } - return await options.wrapUpstreamCall(() => options.fetcher(url, { ...init, headers })); -}; - -const azureFetchInternal = async ( - config: AzureUpstreamConfig, - surface: 'openai' | 'anthropic', - path: string, - init: FetchInit, - options: UpstreamFetchOptions, - query?: string, -): Promise => { - const baseUrl = surface === 'openai' ? azureOpenAiV1BaseUrl(config.endpoint) : azureAnthropicBaseUrl(config.endpoint); - const url = joinBaseAndPath(baseUrl, path); - if (!query) { - return await azureFetchUrl(config, surface, url, init, options); - } - // Append per-endpoint query through URL.searchParams so a future path - // that itself carries a query suffix does not produce `path?a?b`. - const parsed = new URL(url); - for (const [key, value] of new URLSearchParams(query).entries()) parsed.searchParams.append(key, value); - return await azureFetchUrl(config, surface, parsed.href, init, options); -}; - -const azureDeploymentScopedOpenAIAudioTranscriptionUrl = (config: AzureUpstreamConfig, deployment: string): string => { - const url = new URL(config.endpoint); - url.pathname = `/openai/deployments/${encodeURIComponent(deployment)}/audio/transcriptions`; - url.search = ''; - url.hash = ''; - url.searchParams.set('api-version', '2025-04-01-preview'); - return url.href; -}; - -export const azureFetchOpenAIChatCompletions = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/chat/completions', init, options); -export const azureFetchOpenAIResponses = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/responses', init, options); -export const azureFetchOpenAIResponsesCompact = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/responses/compact', init, options); -export const azureFetchOpenAIEmbeddings = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/embeddings', init, options); -export const azureFetchOpenAICompletions = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/completions', init, options); -// gpt-image-2 (released 2026-04-21) and the gpt-image-1 family are exposed -// only under Azure's preview lifecycle today. We will drop the query suffix -// once Azure promotes the image endpoints to the GA default. -export const azureFetchOpenAIImagesGenerations = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/images/generations', init, options, 'api-version=preview'); -export const azureFetchOpenAIImagesEdits = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/images/edits', init, options, 'api-version=preview'); -// Azure selects the transcription deployment in the operation path, so the -// multipart body must omit `model`. The 2025-04 preview route covers both -// Whisper and GPT transcription deployments; all configured endpoint shapes -// reduce to their resource host before this operation-specific path is set. -// https://github.com/Azure/azure-rest-api-specs/blob/b0a48bcbffead733affe03944ef09f5e8d12f8c8/specification/cognitiveservices/OpenAI.Inference/models/audio/audio_transcription.tsp#L119-L126 -// https://github.com/Azure/azure-rest-api-specs/blob/928047803788f7377fa003a26ba2bdc2e0fcccc0/specification/cognitiveservices/OpenAI.Inference/routes/audio_transcription.tsp#L19-L49 -export const azureFetchOpenAIAudioTranscriptions = (config: AzureUpstreamConfig, deployment: string, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchUrl(config, 'openai', azureDeploymentScopedOpenAIAudioTranscriptionUrl(config, deployment), init, options); -export const azureFetchAnthropicMessages = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'anthropic', '/v1/messages', init, options); -export const azureFetchAnthropicMessagesCountTokens = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'anthropic', '/v1/messages/count_tokens', init, options); diff --git a/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts b/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts deleted file mode 100644 index 84603d8c12..0000000000 --- a/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import { mapOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses'; - -/** - * Azure OpenAI still enforces the former namespace-description minLength even - * though current OpenAI Responses permits an empty required string. Supply a - * deterministic model-facing description only on Azure-bound requests. - * - * References: - * - https://github.com/openai/openai-openapi/commit/466c74a42f51c02f1927bc666815251dc53845dc - * - https://github.com/openai/codex/issues/37380 - */ -export const withEmptyNamespaceDescriptionsFilled = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = mapOpenAIResponsesTools(ctx.payload, tool => tool.type === 'namespace' && tool.description === '' - ? { ...tool, description: `Tools in the ${tool.name} namespace.` } - : tool); - return await run(); -}; diff --git a/packages/provider-azure/src/interceptors/openai-responses/index.ts b/packages/provider-azure/src/interceptors/openai-responses/index.ts deleted file mode 100644 index 53e8066faf..0000000000 --- a/packages/provider-azure/src/interceptors/openai-responses/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { withEmptyNamespaceDescriptionsFilled } from './fill-empty-namespace-descriptions.ts'; -import type { AzureOpenAIResponsesBoundaryInterceptor } from './types.ts'; - -// Azure-specific OpenAI Responses workarounds stay behind the provider boundary so -// the gateway and OpenAI-compatible providers retain the canonical wire value. -export const AZURE_OPENAI_RESPONSES_BOUNDARY = [ - withEmptyNamespaceDescriptionsFilled, -] as const satisfies readonly AzureOpenAIResponsesBoundaryInterceptor[]; diff --git a/packages/provider-azure/src/interceptors/openai-responses/types.ts b/packages/provider-azure/src/interceptors/openai-responses/types.ts deleted file mode 100644 index 776def1da3..0000000000 --- a/packages/provider-azure/src/interceptors/openai-responses/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { Interceptor } from '@floway-dev/interceptor'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import type { ProviderModel, ProviderOpenAIResponsesResult, OpenAIResponsesAction } from '@floway-dev/provider'; - -export interface OpenAIResponsesBoundaryCtx { - payload: CanonicalOpenAIResponsesPayload; - headers: Headers; - readonly model: ProviderModel; - action: OpenAIResponsesAction; -} - -export type AzureOpenAIResponsesBoundaryInterceptor = Interceptor< - OpenAIResponsesBoundaryCtx, - ProviderOpenAIResponsesResult ->; diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index 2442e46cb1..3a9b4df46f 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -1,53 +1,13 @@ import { assertAzureUpstreamRecord } from './config.ts'; import { AZURE_DEFAULT_FLAGS } from './defaults.ts'; -import { azureFetchOpenAIAudioTranscriptions, azureFetchOpenAIChatCompletions, azureFetchOpenAICompletions, azureFetchOpenAIEmbeddings, azureFetchOpenAIImagesEdits, azureFetchOpenAIImagesGenerations, azureFetchAnthropicMessages, azureFetchAnthropicMessagesCountTokens, azureFetchOpenAIResponses, azureFetchOpenAIResponsesCompact } from './fetch.ts'; -import { AZURE_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; -import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { createAzurePipelines } from './pipelines.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { kindForEndpoints } from '@floway-dev/protocols/common'; -import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; -import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResult, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; -import { headersForAnthropicMessagesCall, jsonRequestBody, serializeModelPathOpenAIAudioTranscriptionRequest, serializeOpenAIImagesEditsRequest, type FetchInit, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; - -const upstreamModelIdOf = (model: ProviderModel): string => (model.providerData as { upstreamModelId: string }).upstreamModelId; - -type AzureTypedFetch = (config: ReturnType['config'], init: FetchInit, options: UpstreamFetchOptions) => Promise; +import { publicModelId, resolveEffectiveFlags, type ProviderInstance, type Provider, type UpstreamRecord } from '@floway-dev/provider'; export const createAzureProvider = (record: UpstreamRecord): Provider => { const azure = assertAzureUpstreamRecord(record); - const callStreaming = ( - transport: AzureTypedFetch, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - parser: ProviderStreamParser, - opts: UpstreamCallOptions, - ) => { - const upstreamModelId = upstreamModelIdOf(model); - return streamingProviderCall( - transport( - azure.config, - { method: 'POST', body: jsonRequestBody({ ...body, stream: true, model: upstreamModelId }), signal }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ), - parser, - upstreamModelId, - signal, - ); - }; - - const callNonStreaming = async (transport: AzureTypedFetch, model: ProviderModel, body: Record, signal: AbortSignal | undefined, headers: HttpHeaderLines, opts: UpstreamCallOptions) => { - const upstreamModelId = upstreamModelIdOf(model); - const response = await transport(azure.config, { method: 'POST', body: jsonRequestBody({ ...body, model: upstreamModelId }), signal }, { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: upstreamModelId }; - }; - const instance: ProviderInstance = { - callAlphaSearch: () => Promise.reject(new Error('Azure provider does not support callAlphaSearch')), getProvidedModels() { return Promise.resolve(azure.config.models.map(model => { const effective = resolveEffectiveFlags([AZURE_DEFAULT_FLAGS, azure.flagOverrides, model.flagOverrides]); @@ -68,60 +28,6 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { }; })); }, - callOpenAICompletions: (model, body, signal, opts) => callNonStreaming(azureFetchOpenAICompletions, model, body, signal, [...opts.headers], opts), - callOpenAIChatCompletions: (model, body, signal, opts) => callStreaming(azureFetchOpenAIChatCompletions, model, body, signal, [...opts.headers], parseOpenAIChatCompletionsStream, opts), - callOpenAIResponses: async (model, body, action, signal, opts) => { - const ctx: OpenAIResponsesBoundaryCtx = { - payload: { ...body, model: model.id }, - headers: new Headers(opts.headers), - model, - action, - }; - return await runInterceptors( - ctx, AZURE_OPENAI_RESPONSES_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - switch (ctx.action) { - case 'generate': { - const stream = await callStreaming(azureFetchOpenAIResponses, model, wireBody, signal, [...ctx.headers], parseOpenAIResponsesStream, opts); - return stream.ok - ? { action: 'generate', ok: true, events: stream.events, modelKey: stream.modelKey, ...(stream.headers ? { headers: stream.headers } : {}) } - : { action: 'generate', ok: false, response: stream.response, modelKey: stream.modelKey }; - } - case 'compact': { - const upstreamModelId = upstreamModelIdOf(model); - const response = await azureFetchOpenAIResponsesCompact( - azure.config, - { method: 'POST', body: jsonRequestBody({ ...toCompactPayloadShape(wireBody), model: upstreamModelId }), signal }, - { extraHeaders: [...ctx.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return response.ok - ? { action: 'compact', ok: true, result: (await response.json()) as OpenAIResponsesCompactionResult, modelKey: upstreamModelId } - : { action: 'compact', ok: false, response, modelKey: upstreamModelId }; - } - default: - ctx.action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${ctx.action as string}`); - } - }, - ); - }, - callAnthropicMessages: (model, body, signal, opts) => callStreaming(azureFetchAnthropicMessages, model, body, signal, headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta), parseAnthropicMessagesStream, opts), - callAnthropicMessagesCountTokens: (model, body, signal, opts) => callNonStreaming(azureFetchAnthropicMessagesCountTokens, model, body, signal, headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta), opts), - callOpenAIEmbeddings: (model, body, signal, opts) => callNonStreaming(azureFetchOpenAIEmbeddings, model, body, signal, [...opts.headers], opts), - callOpenAIImagesGenerations: (model, body, signal, opts) => callNonStreaming(azureFetchOpenAIImagesGenerations, model, body, signal, [...opts.headers], opts), - callOpenAIImagesEdits: async (model, request, signal, opts) => { - const upstreamModelId = upstreamModelIdOf(model); - const body = await serializeOpenAIImagesEditsRequest(request, upstreamModelId); - const response = await azureFetchOpenAIImagesEdits(azure.config, { method: 'POST', body, signal }, { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: upstreamModelId }; - }, - callOpenAIAudioTranscriptions: async (model, request, signal, opts) => { - const upstreamModelId = upstreamModelIdOf(model); - const body = serializeModelPathOpenAIAudioTranscriptionRequest(request); - const response = await azureFetchOpenAIAudioTranscriptions(azure.config, upstreamModelId, { method: 'POST', body, signal }, { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: upstreamModelId }; - }, - callRerank: () => Promise.reject(new Error('Azure provider does not support callRerank')), }; return { diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/detect-claude-code-metadata_test.ts b/packages/provider-copilot/__tests__/claude-code-metadata_test.ts similarity index 94% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/detect-claude-code-metadata_test.ts rename to packages/provider-copilot/__tests__/claude-code-metadata_test.ts index 5cc0be8dd1..856ff19f48 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/detect-claude-code-metadata_test.ts +++ b/packages/provider-copilot/__tests__/claude-code-metadata_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { parseUserIdMetadata } from '../../../src/interceptors/anthropic-messages/detect-claude-code-metadata.ts'; +import { parseUserIdMetadata } from '../src/claude-code-metadata.ts'; import { assertEquals } from '@floway-dev/test-utils'; test('parseUserIdMetadata returns nulls for undefined input', () => { diff --git a/packages/provider-copilot/__tests__/interceptors/image-size_test.ts b/packages/provider-copilot/__tests__/image-size_test.ts similarity index 93% rename from packages/provider-copilot/__tests__/interceptors/image-size_test.ts rename to packages/provider-copilot/__tests__/image-size_test.ts index 28276fa1f7..4f1e9f19e6 100644 --- a/packages/provider-copilot/__tests__/interceptors/image-size_test.ts +++ b/packages/provider-copilot/__tests__/image-size_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { targetSizeForOpenAIResponsesChat } from '../../src/interceptors/image-size.ts'; +import { targetSizeForOpenAIResponsesChat } from '../src/image-size.ts'; import { assertEquals } from '@floway-dev/test-utils'; test('gpt-4o / gpt-4.1 use the 2048 box + 768 short edge', () => { diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts deleted file mode 100644 index aa2daf658a..0000000000 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { test } from 'vitest'; - -import { CLAUDE_AGENT_USER_AGENT } from '../../../src/auth.ts'; -import { COPILOT_ANTHROPIC_MESSAGES_BOUNDARY } from '../../../src/interceptors/anthropic-messages/index.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ - payload, - headers: new Headers(), - anthropicBeta: [], - model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), -}); - -const COMPACT_LAST_MESSAGE_TEXT = - 'Your task is to create a detailed summary of the conversation so far.\n\n' + - 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.\n\n' + - 'Pending Tasks:\n- finish refactor\n\nCurrent Work:\n- reviewing diff'; - -test('Claude Code SDK compact request: Claude-agent overrides compact intent, both halves of metadata threaded through', async () => { - // This is the realistic ordering case: a Claude Code compact summary call - // ALSO carries the Claude Code SDK fingerprint. We expect the final wire - // headers to be `messages-proxy` (Claude-agent wins over compact's - // `conversation-compaction`), the user-agent and integration-id deletion - // from Claude-agent, and an `x-interaction-id` from the interaction-id - // interceptor — matching what VSCode Copilot Chat sends for the same call. - const ctx = invocation({ - model: 'claude-test', - max_tokens: 10, - metadata: { user_id: JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }) }, - messages: [{ role: 'user', content: COMPACT_LAST_MESSAGE_TEXT }], - }); - - await runInterceptors>>( - ctx, - COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, - okEvents, - ); - - // Compact set `x-initiator: agent` early; `withInitiatorHeaderSet` runs - // later in the merged boundary chain and re-derives x-initiator from the - // last-message structure, so the final wire value reflects the wire-shape - // pass. (That is the same value the pre-merge production code shipped, - // because the target chain always overrode the source-side tag.) - assertEquals(ctx.headers.get('x-initiator'), 'user'); - // Compact set `conversation-compaction`; Claude-agent's `messages-proxy` - // runs after and overrides it. This mirrors caozhiyuan/copilot-api's - // prepareForCompact → prepareMessageProxyHeaders order. - assertEquals(ctx.headers.get('x-interaction-type'), 'messages-proxy'); - assertEquals(ctx.headers.get('openai-intent'), 'messages-proxy'); - assertEquals(ctx.headers.get('user-agent'), CLAUDE_AGENT_USER_AGENT); - // Empty-string sentinel: `copilotAuthedFetch` in - // `packages/provider-copilot/src/auth.ts` deletes the base header. - assertEquals(ctx.headers.get('copilot-integration-id'), ''); - // SHA-256-then-UUIDv4 of 'sess-1' (matches caozhiyuan's getUUID). - assertEquals(ctx.headers.get('x-interaction-id'), 'abe633f3-a47a-4758-974e-abe9160daf36'); -}); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-carrier_test.ts b/packages/provider-copilot/__tests__/opaque-item-id_test.ts similarity index 95% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-carrier_test.ts rename to packages/provider-copilot/__tests__/opaque-item-id_test.ts index 006d304875..c48c475951 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-carrier_test.ts +++ b/packages/provider-copilot/__tests__/opaque-item-id_test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { unwrapCopilotItemId, wrapCopilotItemId } from '../../../src/interceptors/openai-responses/item-id-carrier.ts'; +import { unwrapCopilotItemId, wrapCopilotItemId } from '../src/opaque-item-id.ts'; describe('Copilot item id carrier', () => { test.each([ diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/pipeline-headers_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/pipeline-headers_test.ts new file mode 100644 index 0000000000..c52c30840d --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/pipeline-headers_test.ts @@ -0,0 +1,31 @@ +import { test } from 'vitest'; + +import { CLAUDE_AGENT_USER_AGENT, clearInProcessCopilotTokenCache } from '../../../src/auth.ts'; +import { createCopilotProvider } from '../../../src/provider.ts'; +import { emptyCopilotUpstreamState } from '../../../src/state.ts'; +import { initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; +import { assertEquals, collectChatProviderPipeline, noopAnthropicMessagesUpstreamCallOptions, sseResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; + +const compactText = 'Your task is to create a detailed summary of the conversation so far.\n\nCRITICAL: Respond with TEXT ONLY. Do NOT call any tools.\n\nPending Tasks:\n- finish refactor\n\nCurrent Work:\n- reviewing diff'; + +test('Claude Code compact intent, identity and interaction metadata reach the final Copilot wire in stage order', async () => { + clearInProcessCopilotTokenCache(); + const record: UpstreamRecord = { id: 'up_header_stages', kind: 'copilot', name: 'Test', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: { githubHost: 'github.com', githubToken: 'test-token', user: { id: 1, login: 'test', name: null, avatar_url: '' } }, state: { ...emptyCopilotUpstreamState(), copilotToken: { token: 'access-token', expiresAt: 4102444800, baseUrl: 'https://copilot.example' } }, flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210 }; + initProviderRepo(() => ({ upstreams: { getById: async () => record, saveState: async () => {} } })); + const provider = createCopilotProvider(record); + const model = stubProviderModel({ id: 'claude-test', providerData: { rawModels: [{ id: 'claude-test', supported_endpoints: ['/v1/messages'] }] } }); + let observed: Headers | undefined; + await withMockedFetch(request => { + observed = request.headers; + return sseResponse('event: message_stop\ndata: {"type":"message_stop"}\n\n'); + }, async () => { + await collectChatProviderPipeline(provider, 'anthropicMessages', model, { max_tokens: 10, metadata: { user_id: JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }) }, messages: [{ role: 'user', content: compactText }] }, undefined, noopAnthropicMessagesUpstreamCallOptions()); + }); + if (observed === undefined) throw new Error('Expected model endpoint dispatch'); + assertEquals(observed.get('x-initiator'), 'user'); + assertEquals(observed.get('x-interaction-type'), 'messages-proxy'); + assertEquals(observed.get('openai-intent'), 'messages-proxy'); + assertEquals(observed.get('user-agent'), CLAUDE_AGENT_USER_AGENT); + assertEquals(observed.get('copilot-integration-id'), null); + assertEquals(observed.get('x-interaction-id'), 'abe633f3-a47a-4758-974e-abe9160daf36'); +}); diff --git a/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts index 5a347e0542..2c6affa59c 100644 --- a/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; -import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/interceptors/shared/whitespace-overflow.ts'; import { copilotOpenAIChatCompletionsAbortToolWhitespace } from '../../../src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts'; +import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/tool-argument-whitespace.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { ExecuteResult } from '@floway-dev/provider'; diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts index c14ffe7984..8123bdce89 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; -import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/interceptors/shared/whitespace-overflow.ts'; import { copilotOpenAIResponsesAbortToolWhitespace } from '../../../src/stages/openai-responses/abort-on-tool-argument-whitespace.ts'; +import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/tool-argument-whitespace.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts index 65788e90a4..6ba3187951 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; -import { unwrapCopilotItemId, wrapCopilotItemId } from '../../../src/interceptors/openai-responses/item-id-carrier.ts'; +import { unwrapCopilotItemId, wrapCopilotItemId } from '../../../src/opaque-item-id.ts'; import { copilotOpenAIResponsesItemIdMembrane } from '../../../src/stages/openai-responses/item-id-membrane.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { openaiResponsesResultToEvents, type OpenAIResponsesInputItem, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; diff --git a/packages/provider-copilot/package.json b/packages/provider-copilot/package.json index d813df5f67..aa5dd56db0 100644 --- a/packages/provider-copilot/package.json +++ b/packages/provider-copilot/package.json @@ -4,7 +4,10 @@ "version": "0.0.0", "type": "module", "exports": { - ".": { "import": "./src/index.ts", "types": "./src/index.ts" } + ".": { + "import": "./src/index.ts", + "types": "./src/index.ts" + } }, "scripts": { "typecheck": "tsc --noEmit" @@ -12,7 +15,6 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/platform": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", diff --git a/packages/provider-copilot/src/auth.ts b/packages/provider-copilot/src/auth.ts index 98a1e4b06c..341c2013c2 100644 --- a/packages/provider-copilot/src/auth.ts +++ b/packages/provider-copilot/src/auth.ts @@ -246,14 +246,14 @@ export const copilotRequestHeaders = (entry: CopilotTokenEntry, initHeaders: Hea headers.set('x-interaction-type', 'conversation-agent'); // Provider-attached invocation headers (vision, initiator, anthropic-beta, - // ...) flow through unchanged. The provider's boundary interceptors decide + // ...) flow through unchanged. The provider's boundary stages decide // which headers each upstream call needs; this layer only knows how to ship - // them. Setting them last lets workaround interceptors override the static + // them. Setting them last lets workaround stages override the static // VSCode identification block when a future workaround needs to. // - // Convention: an empty-string value from an interceptor means "delete this - // base header" — the interceptor wants Copilot to NOT see a default we'd - // otherwise pin. An interceptor that wants to clear an arbitrary downstream + // Convention: an empty-string value from an stage means "delete this + // base header" — the stage wants Copilot to NOT see a default we'd + // otherwise pin. An stage that wants to clear an arbitrary downstream // header value must do so by name through this sentinel; the layer does not // otherwise expose a per-header delete API. if (attached) { diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/detect-claude-code-metadata.ts b/packages/provider-copilot/src/claude-code-metadata.ts similarity index 100% rename from packages/provider-copilot/src/interceptors/anthropic-messages/detect-claude-code-metadata.ts rename to packages/provider-copilot/src/claude-code-metadata.ts diff --git a/packages/provider-copilot/src/control-http.ts b/packages/provider-copilot/src/control-http.ts new file mode 100644 index 0000000000..630039b0d8 --- /dev/null +++ b/packages/provider-copilot/src/control-http.ts @@ -0,0 +1,45 @@ +import { copilotAuthedFetch, isCopilotTokenFetchError, type CopilotAuth } from './auth.ts'; +import { parseCopilotQuotaHeaders, putCopilotQuota } from './quota.ts'; +import type { FetchInit, UpstreamFetchOptions } from '@floway-dev/provider'; + +export type CopilotFetchConfig = CopilotAuth; + +export interface CopilotControlFetchOptions extends UpstreamFetchOptions { + /** Extends control-call persistence beyond the response where the runtime supports it. */ + waitUntil?: (promise: Promise) => void; +} + +// Keep any entitlement metadata returned by the catalog refresh without delaying its result. +const captureQuotaFireAndForget = ( + upstreamId: string, + headers: Headers, + waitUntil: ((promise: Promise) => void) | undefined, +): void => { + const snapshot = parseCopilotQuotaHeaders(headers, new Date()); + if (snapshot === null) return; + const persist = putCopilotQuota(upstreamId, snapshot).catch((error: unknown) => { + console.warn(`Failed to persist Copilot quota snapshot for ${upstreamId}:`, error); + }); + waitUntil?.(persist); +}; + +// A rejected credential exchange retains its HTTP reply for catalog error diagnostics. +export const copilotFetchModels = async ( + config: CopilotFetchConfig, + init: FetchInit, + options: CopilotControlFetchOptions, +): Promise => { + const response = await copilotAuthedFetch('/models', init, config, { + headers: options.extraHeaders, + fetcher: options.fetcher, + wrapUpstreamCall: options.wrapUpstreamCall, + }).catch(error => { + if (!isCopilotTokenFetchError(error)) throw error; + return new Response(error.body, { + status: error.status, + headers: new Headers(error.headers), + }); + }); + captureQuotaFireAndForget(config.id, response.headers, options.waitUntil); + return response; +}; diff --git a/packages/provider-copilot/src/fetch-models.ts b/packages/provider-copilot/src/fetch-models.ts index 6614b79b7a..04a65d9f01 100644 --- a/packages/provider-copilot/src/fetch-models.ts +++ b/packages/provider-copilot/src/fetch-models.ts @@ -1,4 +1,4 @@ -import { copilotFetchModels, type CopilotFetchConfig } from './fetch.ts'; +import { copilotFetchModels, type CopilotFetchConfig } from './control-http.ts'; import type { CopilotModelsResponse } from './types.ts'; import { fetchUpstreamModels, type Fetcher, type HttpHeaderLines, identityWrapUpstreamCall } from '@floway-dev/provider'; diff --git a/packages/provider-copilot/src/fetch.ts b/packages/provider-copilot/src/fetch.ts deleted file mode 100644 index a7d2449545..0000000000 --- a/packages/provider-copilot/src/fetch.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { copilotAuthedFetch, isCopilotTokenFetchError, type CopilotAuth } from './auth.ts'; -import { parseCopilotQuotaHeaders, putCopilotQuota } from './quota.ts'; -import type { FetchInit, UpstreamFetchOptions } from '@floway-dev/provider'; - -export type CopilotFetchConfig = CopilotAuth; - -export interface CopilotDataPlaneFetchOptions extends UpstreamFetchOptions { - /** Extends the runtime past the response so the quota persist below - * survives on workerd, which cancels orphan promises the moment the client - * response is sent. The catalog fetch threads none: a `/models` response - * carries no quota headers, so nothing is ever scheduled on that path. */ - waitUntil?: (promise: Promise) => void; -} - -// Every Copilot data-plane response carries the seat's entitlement in -// `x-quota-snapshot-*`, so this funnel — the one place all of them pass -// through — is where the snapshot gets harvested. Streaming included: the -// headers land ahead of the first SSE byte. -// -// Best-effort by construction. The response is already the caller's to relay, -// and a snapshot is a strictly better-than-nothing dashboard read: a CAS loss -// to a concurrent state write, or a storage blip, must not surface on the hot -// path. A response with no quota headers (every 4xx we have observed, plus -// `/models`) leaves the persisted snapshot alone rather than erasing it. -const captureQuotaFireAndForget = ( - upstreamId: string, - headers: Headers, - waitUntil: ((promise: Promise) => void) | undefined, -): void => { - const snapshot = parseCopilotQuotaHeaders(headers, new Date()); - if (snapshot === null) return; - const persist = putCopilotQuota(upstreamId, snapshot).catch((error: unknown) => { - console.warn(`Failed to persist Copilot quota snapshot for ${upstreamId}:`, error); - }); - waitUntil?.(persist); -}; - -// Token-exchange failures surface as regular OpenAI Responses so callers handle them via the same 4xx/5xx path. -const copilotFetchInternal = async ( - config: CopilotFetchConfig, - path: string, - init: FetchInit, - options: CopilotDataPlaneFetchOptions, -): Promise => { - const response = await copilotAuthedFetch(path, init, config, { - headers: options.extraHeaders, - fetcher: options.fetcher, - wrapUpstreamCall: options.wrapUpstreamCall, - }).catch(error => { - if (!isCopilotTokenFetchError(error)) throw error; - return new Response(error.body, { - status: error.status, - headers: new Headers(error.headers), - }); - }); - captureQuotaFireAndForget(config.id, response.headers, options.waitUntil); - return response; -}; - -export const copilotFetchOpenAIChatCompletions = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/chat/completions', init, options); -export const copilotFetchOpenAIResponses = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/responses', init, options); -export const copilotFetchAnthropicMessages = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/v1/messages', init, options); -export const copilotFetchAnthropicMessagesCountTokens = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/v1/messages/count_tokens', init, options); -export const copilotFetchOpenAIEmbeddings = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/embeddings', init, options); -export const copilotFetchModels = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/models', init, options); diff --git a/packages/provider-copilot/src/interceptors/image-compression.ts b/packages/provider-copilot/src/image-compression.ts similarity index 96% rename from packages/provider-copilot/src/interceptors/image-compression.ts rename to packages/provider-copilot/src/image-compression.ts index 927c4553bb..c1d2625a0b 100644 --- a/packages/provider-copilot/src/interceptors/image-compression.ts +++ b/packages/provider-copilot/src/image-compression.ts @@ -23,7 +23,7 @@ const compressImageDataUrlToWebp = async ( }; // A single agentic request often replays the same screenshot across many turns, -// so the boundary interceptors run `Promise.all` over dozens of inline images +// so the boundary stages run `Promise.all` over dozens of inline images // that hash to the same cache key. Without dedup, every duplicate races a // concurrent cache write on that one key, tripping Cloudflare KV's per-key // 1-write/sec limit and wasting work on the Node target. The returned function diff --git a/packages/provider-copilot/src/interceptors/image-size.ts b/packages/provider-copilot/src/image-size.ts similarity index 100% rename from packages/provider-copilot/src/interceptors/image-size.ts rename to packages/provider-copilot/src/image-size.ts diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts deleted file mode 100644 index 23051edc04..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import type { - AnthropicMessagesAssistantInputContentBlock, - AnthropicMessagesPayload, - AnthropicMessagesTextBlock, - AnthropicMessagesUserContentBlock, -} from '@floway-dev/protocols/anthropic-messages'; - -/** - * Anthropic's Messages API defines a top-level `cache_control` field that - * "automatically applies a cache_control marker to the last cacheable block - * in the request" (per `MessageCreateParamsBase` in the official SDK). - * Copilot's `/v1/messages` deployment validates against an older schema for - * several model slots (claude-haiku-4.5, claude-sonnet-4.5, claude-sonnet-4.6, - * and intermittently claude-opus-4.5) and rejects the top-level field with - * `cache_control: Extra inputs are not permitted`. Newer slots (opus 4.6/4.7) - * silently accept it. - * - * Port the marker onto the last cacheable content block (mirroring the - * documented semantics), then drop the top-level field. If the last cacheable - * block already carries its own `cache_control`, leave it alone — an explicit - * marker wins over the auto-apply. Sub-field extensions (`scope`, `ttl`) - * carried in the ported value are cleaned up by - * `withCacheControlExtensionsStripped`, which runs immediately after. - * - * References: - * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L2909-L2913 - * - https://github.com/caozhiyuan/copilot-api/issues/269 - */ - -type CacheableBlock = Extract< - AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock, - { cache_control?: unknown } ->; - -const isCacheableBlock = (block: AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock): block is CacheableBlock => - block.type === 'text' || block.type === 'image' || block.type === 'tool_use' || block.type === 'tool_result'; - -/** Where the marker lands: the last message that can carry one, and within it either the whole - * string content or the last cacheable block. Reading the position out first is what lets the - * rewrite below rebuild one path and leave every other message alone. */ -const markerSite = (messages: AnthropicMessagesPayload['messages']): { readonly message: number; readonly block: number | 'whole' } | undefined => { - for (let m = messages.length - 1; m >= 0; m--) { - const content = messages[m].content; - if (typeof content === 'string') return { message: m, block: 'whole' }; - for (let b = content.length - 1; b >= 0; b--) { - if (isCacheableBlock(content[b])) return { message: m, block: b }; - } - } - return undefined; -}; - -export const withTopLevelCacheControlApplied: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { - const payload = ctx.payload as typeof ctx.payload & { cache_control?: { type: 'ephemeral' } }; - const topLevel = payload.cache_control; - if (topLevel === undefined) return await run(); - - const { cache_control: _ported, ...withoutTopLevel } = payload; - const site = markerSite(payload.messages); - if (site === undefined) { - ctx.payload = withoutTopLevel; - return await run(); - } - - ctx.payload = { - ...withoutTopLevel, - // Each rewrite keeps its message's own role and only re-homes that role's own blocks, so - // the array is handed back at its element type; spreading alone widens it to the union. - messages: payload.messages.map((message, m): AnthropicMessagesPayload['messages'][number] => { - if (m !== site.message) return message; - if (site.block === 'whole') { - const block: AnthropicMessagesTextBlock = { type: 'text', text: message.content as string, cache_control: topLevel }; - return { ...message, content: [block] } as AnthropicMessagesPayload['messages'][number]; - } - const content = message.content as readonly CacheableBlock[]; - // `??=` semantics: a block that already states its own marker keeps it. - if (content[site.block].cache_control !== undefined) return message; - return { - ...message, - content: content.map((block, b) => (b === site.block ? { ...block, cache_control: topLevel } : block)), - } as AnthropicMessagesPayload['messages'][number]; - }), - }; - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts deleted file mode 100644 index 9e56a357ff..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { memoizedBase64Compressor } from '../image-compression.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import { type ImageSizeCalculator, type SizeCaps, fitWithin } from '@floway-dev/platform'; -import type { AnthropicMessagesImageBlock, AnthropicMessagesMessage, AnthropicMessagesToolResultBlock, AnthropicMessagesToolResultContentBlock, AnthropicMessagesUserContentBlock } from '@floway-dev/protocols/anthropic-messages'; - -// Per-model image caps for the Claude (Anthropic Messages) egress, measured from the real -// /v1/messages generation path (count_tokens misreports the downscale here). -// Copilot's Anthropic Messages endpoint is Anthropic-only by Copilot's own contract, so -// the caps below are Anthropic's canonical vision limits: -// -// - Opus 4.7 was the first high-resolution Claude model; Opus 4.7 and 4.8 -// sample up to ~3.59 MP within a 2576px long edge. -// - Opus 4.5 / 4.6 and the sonnet / haiku families use the standard ~1.18 MP / -// 1568px cap. -// -// We threshold on the Opus version so future Opus releases inherit the high-res -// tier; every non-Opus Claude model uses the standard cap. -// -// References: -// - Anthropic vision docs: https://docs.claude.com/en/docs/build-with-claude/vision -// - High-res Opus 4.7+ sampling: https://docs.claude.com/en/docs/build-with-claude/vision#evaluate-image-size -const STANDARD_CLAUDE_CAPS: SizeCaps = { maxLongEdge: 1568, maxArea: 1_176_000 }; - -const claudeImageCaps = (upstreamModelId: string): SizeCaps => { - const opus = /opus-(\d+)(?:\.(\d+))?/.exec(upstreamModelId); - if (!opus) return STANDARD_CLAUDE_CAPS; - const major = Number(opus[1]); - const minor = opus[2] === undefined ? 0 : Number(opus[2]); - const highRes = major > 4 || (major === 4 && minor >= 7); - return highRes ? { maxLongEdge: 2576, maxArea: 3_588_000 } : STANDARD_CLAUDE_CAPS; -}; - -// Anthropic carries inline images both at the top level of a message's content -// and nested inside tool_result content, mirroring the vision-header scan. -const collectImageBlocks = (messages: AnthropicMessagesMessage[]): AnthropicMessagesImageBlock[] => { - const blocks: AnthropicMessagesImageBlock[] = []; - for (const message of messages) { - if (!Array.isArray(message.content)) continue; - for (const block of message.content) { - if (block.type === 'image') blocks.push(block); - else if (block.type === 'tool_result' && Array.isArray(block.content)) { - for (const inner of block.content) { - if (inner.type === 'image') blocks.push(inner); - } - } - } - } - return blocks; -}; - -const compressInlineImages = async (ctx: AnthropicMessagesBoundaryCtx): Promise => { - const blocks = collectImageBlocks(ctx.payload.messages); - if (blocks.length === 0) return; - - const caps = claudeImageCaps(ctx.model.id); - const targetSize: ImageSizeCalculator = source => fitWithin(source, caps); - const compress = memoizedBase64Compressor(targetSize); - const compressedData = new Map(); - await Promise.all( - blocks.map(async block => { - compressedData.set(block, await compress(block.source.data)); - }), - ); - const hasCompressedImage = (block: AnthropicMessagesToolResultContentBlock | AnthropicMessagesUserContentBlock): block is AnthropicMessagesImageBlock => - block.type === 'image' && compressedData.has(block); - const hasCompressedToolResultImage = ( - block: AnthropicMessagesUserContentBlock, - ): block is AnthropicMessagesToolResultBlock & { content: AnthropicMessagesToolResultContentBlock[] } => - block.type === 'tool_result' - && Array.isArray(block.content) - && block.content.some(hasCompressedImage); - const rewriteImage = (block: AnthropicMessagesImageBlock): AnthropicMessagesImageBlock => { - const data = compressedData.get(block); - if (data === undefined) throw new Error('Missing compressed Anthropic Messages image data'); - return { - ...block, - source: { ...block.source, media_type: 'image/webp', data }, - }; - }; - const rewriteUserContent = (content: AnthropicMessagesUserContentBlock[]): AnthropicMessagesUserContentBlock[] => - content.map(block => { - if (hasCompressedImage(block)) return rewriteImage(block); - if (!hasCompressedToolResultImage(block)) return block; - return { - ...block, - content: block.content.map(inner => hasCompressedImage(inner) ? rewriteImage(inner) : inner), - }; - }); - - ctx.payload = { - ...ctx.payload, - messages: ctx.payload.messages.map(message => - message.role === 'user' - && Array.isArray(message.content) - && message.content.some(block => hasCompressedImage(block) || hasCompressedToolResultImage(block)) - ? { ...message, content: rewriteUserContent(message.content) } - : message), - }; -}; - -// Recompresses every inline base64 image in the outgoing Anthropic Messages payload to -// WebP before the Copilot upstream call. Generic in the run-result type so the -// same definition serves both the streaming Anthropic Messages boundary chain and the -// count_tokens boundary chain, so count_tokens sizes the same recompressed -// payload the chat path sends. -export const withInlineImagesCompressed = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - // Finish this nested activation before starting the upstream call. Its - // request-local memoizer keys are the full source base64 strings, which can - // be several megabytes each and must not stay live for the response stream. - await compressInlineImages(ctx); - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts deleted file mode 100644 index e4d840d865..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; - -/** - * Anthropic Fast Mode is a per-request opt-in carried by `speed: "fast"` on - * the Anthropic Messages request and echoed back as `usage.speed: "fast"`. Copilot - * does not speak Fast Mode on the wire — the upstream resolves the tier via - * the `-fast` raw model id picked by `model-selection.ts` and never echoes - * `usage.speed`. This interceptor bridges the two contracts at the Copilot - * boundary: - * - * - Strip `speed: 'fast' | 'standard'` from the outbound payload. The - * value was already consumed by `callAnthropicMessages` to pick the raw variant - * and pre-validate Fast Mode support, so passing it through to Copilot - * would just trigger an unknown-field 400 from a strict upstream. - * - Leave any other `speed` value untouched. Unknown values mean the - * caller is wrong; let Copilot surface the same invalid_request_error - * Anthropic itself would, rather than the gateway lying about which - * field the upstream rejected. - * - When the caller asked for Fast Mode, stamp `usage.speed = 'fast'` - * onto every `message_start` and `message_delta` frame on the way out - * so downstream sees the marker the billing path (`speed` → tier='fast' - * → the `serviceTier: 'fast'` pricing entry) and Anthropic-compatible clients expect. - * - * References: - * - https://docs.claude.com/en/build-with-claude/fast-mode - * - https://docs.claude.com/en/api/service-tiers - */ -export const withSpeedFast: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { - const speed = ctx.payload.speed; - const stampFast = speed === 'fast'; - - if (speed === 'fast' || speed === 'standard') { - const { speed: _stripped, ...payload } = ctx.payload; - ctx.payload = payload; - } - - const result = await run(); - - if (!stampFast || result.type !== 'events') return result; - return { - ...result, - events: stampFastSpeedOntoUsage(result.events), - }; -}; - -const stampFastSpeedOntoUsage = async function* ( - frames: AsyncIterable>, -): AsyncGenerator> { - for await (const frame of frames) { - if (frame.type === 'done') { - yield frame; - continue; - } - const { event } = frame; - if (event.type === 'message_start') { - yield eventFrame({ - ...event, - message: { - ...event.message, - usage: { ...event.message.usage, speed: 'fast' }, - }, - }); - continue; - } - if (event.type === 'message_delta' && event.usage) { - yield eventFrame({ - ...event, - usage: { ...event.usage, speed: 'fast' }, - }); - continue; - } - yield frame; - } -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts deleted file mode 100644 index da95a92581..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts +++ /dev/null @@ -1,87 +0,0 @@ -// Copilot-only Anthropic Messages workarounds. Each list is a boundary chain the -// Copilot provider runs inside its own `callX` methods, so the gateway main -// flow never knows that Copilot has interceptors at all. - -import { withTopLevelCacheControlApplied } from './apply-top-level-cache-control.ts'; -import { withInlineImagesCompressed } from './compress-images.ts'; -import { withSpeedFast } from './handle-speed-fast.ts'; -import { withAnthropicBetaNormalized } from './normalize-anthropic-beta.ts'; -import { withThinkingDisplayPromoted } from './promote-thinking-display.ts'; -import { rewriteContextWindowError } from './rewrite-context-window-error.ts'; -import { withClaudeAgentHeadersSet } from './set-claude-agent-headers.ts'; -import { withCompactHeadersSet } from './set-compact-headers.ts'; -import { withInitiatorHeaderSet } from './set-initiator-header.ts'; -import { withInteractionIdHeaderSet } from './set-interaction-id-header.ts'; -import { withVisionHeaderSet } from './set-vision-header.ts'; -import { withCacheControlExtensionsStripped } from './strip-cache-control-extensions.ts'; -import { withEagerInputStreamingStripped } from './strip-eager-input-streaming.ts'; -import type { CopilotAnthropicMessagesBoundaryInterceptor, CopilotAnthropicMessagesCountTokensBoundaryInterceptor } from './types.ts'; - -// Order rationale, split into two lanes that run back-to-back: -// -// Lane 1 — source-shape header derivation (must read the pre-mutation -// payload): -// - rewriteContextWindowError wraps the whole chain so any upstream context- -// window failure surfaced from the terminal is rewritten into a -// Anthropic-Messages-shaped invalid_request_error before later interceptors see it. -// - withCompactHeadersSet pins the compact/auto-continue intent first. -// - withClaudeAgentHeadersSet then overrides those intents (and the -// user-agent / copilot-integration-id) for Claude Code SDK proxy traffic. -// - withInteractionIdHeaderSet finally sets `x-interaction-id` from the -// same parsed metadata. -// -// Lane 2 — wire-shape mutators followed by header-from-wire derivation: -// Payload mutators run first so the header interceptors see the final -// outgoing payload; withTopLevelCacheControlApplied runs before -// withCacheControlExtensionsStripped so the ported marker on the last -// cacheable block is cleaned in the same pass. withSpeedFast strips the -// client's `speed` field (already consumed by callAnthropicMessages for raw-variant -// selection) and post-`run()` stamps `usage.speed='fast'` onto outbound -// message_start/message_delta events when Fast Mode was requested. The -// header lane closes by normalizing admitted caller beta intent, then adding -// tokens required by the final thinking and context-management shape. -// `withInitiatorHeaderSet` re-derives x-initiator from the final last-message -// structure and may overwrite the -// compact-tagged value above — that mirrors the pre-boundary target-side -// override. -// -// `withAnthropicMessagesWebSearchShim` is intentionally NOT registered here. The shim -// is the gateway's rather than this provider's — it rewrites Anthropic's -// native server tool into an ordinary client tool and runs the search itself, -// which is work no provider boundary does. This provider opts into it by -// listing `anthropic-messages-web-search-shim` in its default flag set (see -// COPILOT_DEFAULT_FLAGS in ../../defaults.ts). -export const COPILOT_ANTHROPIC_MESSAGES_BOUNDARY = [ - rewriteContextWindowError, - withCompactHeadersSet, - withClaudeAgentHeadersSet, - withInteractionIdHeaderSet, - withInlineImagesCompressed, - withSpeedFast, - withThinkingDisplayPromoted, - withTopLevelCacheControlApplied, - withCacheControlExtensionsStripped, - withEagerInputStreamingStripped, - withVisionHeaderSet, - withInitiatorHeaderSet, - withAnthropicBetaNormalized, -] as const satisfies readonly CopilotAnthropicMessagesBoundaryInterceptor[]; - -// /v1/messages/count_tokens is a one-shot HTTP exchange that returns the raw -// upstream Response. The Copilot provider applies vision detection, -// x-initiator classification and normalized caller/payload beta intent to both -// chat and count_tokens. -// -// withInlineImagesCompressed runs first so count_tokens sizes the same -// WebP-recompressed payload the chat path sends, keeping the estimate -// consistent with the real request. The chat boundary's other -// entries stay out: its post-`run()` inspectors cannot be expressed against a -// raw Response at all, and the remaining payload mutators and header setters -// each answer something we observed on the generation endpoint, with no -// equivalent need seen on count_tokens. -export const COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY = [ - withInlineImagesCompressed, - withVisionHeaderSet, - withInitiatorHeaderSet, - withAnthropicBetaNormalized, -] as const satisfies readonly CopilotAnthropicMessagesCountTokensBoundaryInterceptor[]; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts deleted file mode 100644 index b7c88d187b..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; - -const ALLOWED_ANTHROPIC_BETAS = new Set([ - 'interleaved-thinking-2025-05-14', - 'context-management-2025-06-27', - 'advanced-tool-use-2025-11-20', -]); -const INTERLEAVED_THINKING_BETA = 'interleaved-thinking-2025-05-14'; -const CONTEXT_MANAGEMENT_BETA = 'context-management-2025-06-27'; - -// Copilot rejects unknown beta values. Preserve the supported subset of the -// caller's Anthropic Messages beta intent, synthesize VS Code's thinking default only -// when the caller supplied no beta intent, and keep context_management paired -// with its required token. -// https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/platform/endpoint/node/chatEndpoint.ts#L262-L282 -// https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/extension/chatSessions/claude/node/claudeLanguageModelServer.ts#L413-L427 -export const withAnthropicBetaNormalized = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - const callerSuppliedBeta = ctx.anthropicBeta.length > 0; - ctx.anthropicBeta = [...new Set(ctx.anthropicBeta.filter(beta => ALLOWED_ANTHROPIC_BETAS.has(beta)))]; - - const isAdaptiveThinking = ctx.payload.thinking?.type === 'adaptive'; - if (!callerSuppliedBeta && ctx.payload.thinking?.budget_tokens && !isAdaptiveThinking) { - ctx.anthropicBeta.push(INTERLEAVED_THINKING_BETA); - } - - const payload = ctx.payload as typeof ctx.payload & { context_management?: unknown }; - if (payload.context_management !== undefined && !ctx.anthropicBeta.includes(CONTEXT_MANAGEMENT_BETA)) { - ctx.anthropicBeta.push(CONTEXT_MANAGEMENT_BETA); - } - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts deleted file mode 100644 index bc406b93d1..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts +++ /dev/null @@ -1,114 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx, CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import { copilotRawModelId } from '../../model-name.ts'; -import type { AnthropicMessagesStreamEvent, AnthropicMessagesThinkingDisplay } from '@floway-dev/protocols/anthropic-messages'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; - -const CLAUDE_VERSION_PATTERN = /(?:^|-)(\d+)\.(\d+)(?=-|$)/; - -const isAnthropicMessagesThinkingDisplay = (value: unknown): value is AnthropicMessagesThinkingDisplay => value === 'omitted' || value === 'summarized' || value === 'full'; - -const isClaudeVersionAtLeast = (model: string, major: number, minor: number): boolean => { - const normalized = copilotRawModelId(model); - if (!normalized.startsWith('claude-')) return false; - - const match = normalized.match(CLAUDE_VERSION_PATTERN); - if (!match) return false; - - const modelMajor = Number(match[1]); - const modelMinor = Number(match[2]); - - return modelMajor > major || (modelMajor === major && modelMinor >= minor); -}; - -export const resolveAnthropicMessagesDownstreamThinkingDisplay = (ctx: Pick): AnthropicMessagesThinkingDisplay | undefined => { - const display = ctx.payload.thinking?.display; - if (display !== undefined) { - // Request JSON is not runtime-validated before boundary interceptors; leave - // unknown display values untouched so upstream, not this workaround, owns - // rejecting or accepting future values. - return isAnthropicMessagesThinkingDisplay(display) ? display : undefined; - } - - return isClaudeVersionAtLeast(ctx.payload.model, 4, 7) ? 'omitted' : 'summarized'; -}; - -const omitThinkingTextFromProtocolFrame = (frame: ProtocolFrame): ProtocolFrame | undefined => { - if (frame.type === 'done') return frame; - - const { event } = frame; - if (event.type === 'content_block_start' && event.content_block.type === 'thinking') { - return eventFrame({ - ...event, - content_block: { - ...event.content_block, - thinking: '', - }, - }); - } - - if (event.type === 'content_block_delta' && event.delta.type === 'thinking_delta') { - return undefined; - } - - return frame; -}; - -const omitThinkingTextFromProtocolFrames = async function* (frames: AsyncIterable>): AsyncGenerator> { - for await (const frame of frames) { - const omitted = omitThinkingTextFromProtocolFrame(frame); - if (omitted) yield omitted; - } -}; - -/** - * Workaround for Copilot/Claude turns that go silent during long extended - * thinking, then surface to clients as `API Error: Network connection lost`, - * `Stream idle timeout`, or a short no-tool-call turn. The direct Copilot - * references below document a roughly 60s HTTP response-idle boundary in - * GitHub Copilot paths; the related client reports show the same user-visible - * stall/no-output failure shape in official Copilot and Claude Code clients. - * - * The native Copilot Anthropic Messages target is the boundary where `thinking.display` - * controls whether the upstream emits token-level `thinking_delta` SSE while - * the model is reasoning. Our Copilot probes found Claude 4.7 defaults to - * omitted display, while 4.6/4.5 default to summarized; forcing summarized - * upstream keeps data flowing during thinking and avoids the idle gap. To keep - * downstream omitted semantics, this boundary interceptor removes only thinking - * text/deltas after the upstream attempt and preserves every `signature` byte; - * the same probes showed blank thinking text is accepted, while any signature - * tampering makes the next Anthropic Messages request fail with 400. Those probes justify - * the request/response mechanics here; the public references justify why this - * workaround exists. - * - * References: - * Direct Copilot HTTP idle-boundary reports: - * - https://github.com/ericc-ch/copilot-api/issues/223 - * - https://github.com/copilot-extensions/user-feedback/issues/2 - * Related official-client and downstream symptoms: - * - https://github.com/microsoft/vscode-copilot-release/issues/7640 - * - https://github.com/github/copilot-cli/issues/686 - * - https://github.com/github/copilot-cli/issues/1614 - * - https://github.com/anthropics/claude-code/issues/46987 - * - https://github.com/anthropics/claude-code/issues/50477 - */ -export const withThinkingDisplayPromoted: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { - const downstreamDisplay = resolveAnthropicMessagesDownstreamThinkingDisplay(ctx); - const thinking = ctx.payload.thinking; - const hasActiveThinking = !!thinking && thinking.type !== 'disabled'; - const shouldExposeOmitted = hasActiveThinking && downstreamDisplay === 'omitted'; - - if (hasActiveThinking && downstreamDisplay !== undefined && downstreamDisplay !== 'full') { - ctx.payload.thinking = { - ...thinking, - display: 'summarized', - }; - } - - const result = await run(); - - if (!shouldExposeOmitted || result.type !== 'events') return result; - return { - ...result, - events: omitThinkingTextFromProtocolFrames(result.events), - }; -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts deleted file mode 100644 index 9b8914c8c2..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import { buildPromptTooLongBody } from '@floway-dev/protocols/anthropic-messages'; - -const isContextWindowError = (text: string): boolean => text.includes('Request body is too large for model context window') || text.includes('context_length_exceeded'); - -/** - * Copilot's `/v1/messages` endpoint reports context-window failures with an - * Anthropic-shape body carrying a Copilot-specific message string; Claude - * Code's detector matches on the message substring alone (case-insensitive - * `error.message.toLowerCase().includes('prompt is too long')`), so we - * rewrite this body to the canonical prompt-too-long envelope and trigger - * auto-compaction. The envelope + client-bundle evidence live in - * `@floway-dev/protocols/anthropic-messages` (`buildPromptTooLongBody` / - * `PROMPT_TOO_LONG_MESSAGE`). This interceptor exists in addition to the - * translate-layer rewriter because Copilot's Anthropic Messages endpoint never - * traverses the `anthropic-messages-via-*` translation pairs — the Copilot Anthropic Messages - * substring set here (`Request body is too large...`) is disjoint from - * the OpenAI Responses/OpenAI Chat Completions shapes those pairs match on. - */ -export const rewriteContextWindowError: CopilotAnthropicMessagesBoundaryInterceptor = async (_ctx, run) => { - const result = await run(); - if (result.type !== 'api-error' || result.source !== 'upstream') return result; - - const body = new TextDecoder().decode(result.body); - if (!isContextWindowError(body)) return result; - - return { - ...result, - status: 400, - headers: new Headers({ 'content-type': 'application/json' }), - body: buildPromptTooLongBody(), - }; -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts deleted file mode 100644 index edf4f32297..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { parseUserIdMetadata } from './detect-claude-code-metadata.ts'; -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import { CLAUDE_AGENT_USER_AGENT } from '../../auth.ts'; - -/** - * When Anthropic Messages traffic comes from the Claude Code SDK proxy, VSCode - * Copilot Chat re-tags the native Anthropic Messages upstream call with a different - * intent + user-agent and drops the `copilot-integration-id` it would otherwise - * pin to `vscode-chat`. We mirror that tagging only when the planner selected - * a native Anthropic Messages target and both halves of the Claude Code - * `metadata.user_id` fingerprint are present. - * - * Detection requires BOTH safetyIdentifier AND sessionId so we never apply - * the messages-proxy intent to ordinary chat traffic that happens to share - * one half of the legacy regex. - * - * Sentinel: an empty-string value tells `copilotAuthedFetch` in - * `packages/provider-copilot/src/auth.ts` to delete the named base header. We - * use it to clear `copilot-integration-id` because VSCode Copilot Chat omits - * that header on Claude Code SDK proxy traffic. - * - * Do not put this identity on translated OpenAI Chat Completions / OpenAI Responses targets. - * The real VS Code path forces an Anthropic Messages API request, and caozhiyuan's gateway - * applies the same helper only in its `/v1/messages` Copilot caller. Copilot's - * OpenAI Chat Completions endpoint treats the full messages-proxy + Claude Code UA + no-integration - * shape as `integrator: claude-code`, which can hide non-Claude OpenAI Chat Completions models. - * - * `claude-opus-4-8` is currently excluded: that model's upstream WAF returns - * a generic 403 ("Access to this endpoint is forbidden") whenever a request - * carries the Claude-Code-style user-agent without a `copilot-integration-id` - * header. The same header set is accepted on `claude-opus-4-7`, so the gate - * is a model-id rollout gap on Copilot's side, not a policy choice on ours. - * Skipping the rewrite for 4-8 keeps the default Copilot identity (vscode-chat - * integration id + GitHubCopilotChat UA + conversation-agent intent) in - * place; that path is 200. TODO: remove this skip once Copilot's upstream - * accepts the Claude-Code identity on 4-8. Probed 2026-05-29. - * - * References: - * - https://github.com/microsoft/vscode-copilot-chat/blob/5863f5a7088958050792b5dccbe8b46c6e13eccc/src/extension/chatSessions/claude/node/claudeLanguageModelServer.ts#L479-L516 - * - https://github.com/caozhiyuan/copilot-api/blob/88840ed80000635902b90a35989b1e795d289fdf/src/services/copilot/create-messages.ts#L110-L116 - * - https://github.com/caozhiyuan/copilot-api/blob/88840ed80000635902b90a35989b1e795d289fdf/src/services/copilot/create-chat-completions.ts#L45-L61 - */ -const UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY = new Set(['claude-opus-4-8']); - -export const withClaudeAgentHeadersSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { - if (UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY.has(ctx.payload.model)) { - return await run(); - } - - const { safetyIdentifier, sessionId } = parseUserIdMetadata(ctx.payload.metadata?.user_id); - if (safetyIdentifier && sessionId) { - ctx.headers.set('x-interaction-type', 'messages-proxy'); - ctx.headers.set('openai-intent', 'messages-proxy'); - ctx.headers.set('user-agent', CLAUDE_AGENT_USER_AGENT); - ctx.headers.set('copilot-integration-id', ''); - } - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts deleted file mode 100644 index e9ec7876c3..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts +++ /dev/null @@ -1,126 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import type { AnthropicMessagesMessage, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; - -/** - * Claude Code (and OpenCode) periodically asks the model to summarize the - * conversation so the next turn can run against a compacted transcript. VSCode - * Copilot Chat tags those calls with a different intent + interaction-type so - * Copilot's accounting and abuse controls treat the call as a compaction step - * instead of a user-facing turn. Separately, both agents also resume from a - * fresh out-of-context cut by sending an "auto-continue" prompt; VSCode marks - * those as agent-initiated (without the compaction interaction-type) so they - * still bill as continuation rather than a brand-new user turn. We mirror both - * tagging shapes when we detect them, so Copilot-side billing attribution - * stays consistent with what real VSCode users produce. - * - * Detection is purely structural — we never read auth or per-tenant metadata. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/compact.ts - * - https://github.com/caozhiyuan/copilot-api/blob/main/src/routes/messages/preprocess.ts (getCompactType) - * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/api-config.ts (prepareForCompact) - */ - -// Constants ported verbatim from caozhiyuan/copilot-api so detection drift -// stays observable when their values move. -const COMPACT_TEXT_ONLY_GUARD = 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.'; -const COMPACT_SUMMARY_PROMPT_START = 'Your task is to create a detailed summary of the conversation so far'; -const COMPACT_MESSAGE_SECTIONS = ['Pending Tasks:', 'Current Work:'] as const; -const COMPACT_SYSTEM_PROMPT_STARTS = [ - 'You are a helpful AI assistant tasked with summarizing conversations', - 'You are an anchored context summarization assistant for coding sessions.', -] as const; - -// Auto-continue prompts: the first user turn after Claude Code / OpenCode -// resume from an out-of-context cut. The text comes from the agent harness, -// not the human, so VSCode-shaped accounting treats it as agent-initiated. -// See caozhiyuan's `compactAutoContinuePromptStarts` for the source list. -const COMPACT_AUTO_CONTINUE_PROMPT_STARTS = [ - // Claude Code resume prompt. - 'This session is being continued from a previous conversation that ran out of context. The summary below covers the earlier portion of the conversation.', - // OpenCode primary continuation prompt. - 'Continue if you have next steps, or stop and ask for clarification if you are unsure how to proceed.', - // OpenCode media-eviction continuation prompt. - "The previous request exceeded the provider's size limit due to large media attachments. The conversation was compacted and media files were removed from context.", -] as const; - -// Compact-summary and auto-continue prompts are always authored as a fresh -// user turn — the agent harness injects them on the human's behalf. An -// assistant history item whose text happens to start with one of those -// markers (e.g. a client that round-trips the previous request's user turn -// back as assistant history) must not trip compact tagging, so the extractor -// returns empty text for any non-user role. This mirrors caozhiyuan's -// `getCompactCandidateText`: -// https://github.com/caozhiyuan/copilot-api/blob/main/src/routes/messages/preprocess.ts#L94 -// -// `` blocks are Claude Code's own injected reminders that -// the agent never authored; they should never count as compact-summary -// evidence either. -const lastMessageText = (message: AnthropicMessagesMessage): string => { - if (message.role !== 'user') return ''; - if (typeof message.content === 'string') return message.content; - return message.content - .filter((block): block is AnthropicMessagesTextBlock => block.type === 'text') - .map(block => (block.text.startsWith('') ? '' : block.text)) - .filter(text => text.length > 0) - .join('\n\n'); -}; - -const isCompactLastMessage = (message: AnthropicMessagesMessage | undefined): boolean => { - if (!message) return false; - const text = lastMessageText(message); - if (!text) return false; - // All three markers must be present together; the text-only guard alone - // appears in unrelated Claude Code prompts and would over-match. - return text.includes(COMPACT_TEXT_ONLY_GUARD) && text.includes(COMPACT_SUMMARY_PROMPT_START) && COMPACT_MESSAGE_SECTIONS.some(section => text.includes(section)); -}; - -const isAutoContinueLastMessage = (message: AnthropicMessagesMessage | undefined): boolean => { - if (!message) return false; - const text = lastMessageText(message); - if (!text) return false; - return COMPACT_AUTO_CONTINUE_PROMPT_STARTS.some(prefix => text.startsWith(prefix)); -}; - -const startsWithCompactSystemPrompt = (text: string): boolean => COMPACT_SYSTEM_PROMPT_STARTS.some(prefix => text.startsWith(prefix)); - -const isCompactSystemPrompt = (system: string | AnthropicMessagesTextBlock[] | undefined): boolean => { - if (typeof system === 'string') return startsWithCompactSystemPrompt(system); - if (Array.isArray(system)) return system.some(block => startsWithCompactSystemPrompt(block.text)); - return false; -}; - -type CompactClass = 'compact-request' | 'auto-continue' | null; - -// Match caozhiyuan's `getCompactType` priority exactly: last-message -// compact-summary wins, then last-message auto-continue, then system-prompt -// compact-summary. This ordering matters because a single payload could in -// principle match more than one shape, and the COMPACT_REQUEST tagging is -// strictly stronger than the auto-continue tagging. -const classifyCompact = (payload: { messages: AnthropicMessagesMessage[]; system?: string | AnthropicMessagesTextBlock[] }): CompactClass => { - const last = payload.messages.at(-1); - if (isCompactLastMessage(last)) return 'compact-request'; - if (isAutoContinueLastMessage(last)) return 'auto-continue'; - if (isCompactSystemPrompt(payload.system)) return 'compact-request'; - return null; -}; - -export const withCompactHeadersSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { - const kind = classifyCompact(ctx.payload); - if (kind === 'compact-request') { - ctx.headers.set('x-initiator', 'agent'); - ctx.headers.set('x-interaction-type', 'conversation-compaction'); - // openai-intent stays at `copilotAuthedFetch`'s `conversation-agent` - // default from `packages/provider-copilot/src/auth.ts` — the same value - // caozhiyuan/copilot-api re-pins inside prepareForCompact, so setting it - // here would be a no-op. - } else if (kind === 'auto-continue') { - // Auto-continue gets only the agent-initiator tag; interaction-type stays - // at `copilotAuthedFetch`'s `conversation-agent` default from - // `packages/provider-copilot/src/auth.ts`. - // This mirrors prepareForCompact when compactType === COMPACT_AUTO_CONTINUE: - // it sets x-initiator: agent and leaves x-interaction-type untouched. - ctx.headers.set('x-initiator', 'agent'); - } - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts deleted file mode 100644 index aae4ccf536..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; - -/** - * Copilot's `x-initiator` header distinguishes turns that the human user just - * triggered (`user`) from turns the agent triggered to consume a tool result - * (`agent`). The header gates Copilot-side abuse controls and conversation - * accounting. We classify by the last message: a user turn whose content - * mixes plain blocks with tool_results is still user-initiated; a user turn - * whose content is *only* tool_results is the agent consuming results, and - * any assistant-final turn (which only happens on count-tokens replays) is - * always agent. - * - * The header name is lowercase `x-initiator`; HTTP header names are - * case-insensitive on the wire, so the casing is cosmetic. - * - * Generic in the run-result type so the count_tokens boundary chain - * (`Response`) and the streaming Anthropic Messages boundary chain (`ExecuteResult<...>`) - * can share one definition, matching the established behavior where - * x-initiator is set on every Copilot Anthropic Messages HTTP call. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/master/src/services/copilot/create-chat-completions.ts - */ -export const withInitiatorHeaderSet = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - const lastMessage = ctx.payload.messages[ctx.payload.messages.length - 1]; - let initiator: 'user' | 'agent'; - if (lastMessage?.role !== 'user') { - initiator = 'agent'; - } else if (!Array.isArray(lastMessage.content)) { - initiator = 'user'; - } else { - initiator = lastMessage.content.some(block => block.type !== 'tool_result') ? 'user' : 'agent'; - } - ctx.headers.set('x-initiator', initiator); - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts deleted file mode 100644 index 2dfa0cfc08..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { v4 } from 'uuid'; - -import { parseUserIdMetadata } from './detect-claude-code-metadata.ts'; -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; - -/** - * Copilot's `x-interaction-id` header threads a conversation through its - * accounting and trace tooling. We hash the raw session identifier through - * SHA-256 and format the first 16 bytes as a UUID v4 string, so the on-wire - * value stays a UUID-shaped opaque identifier rather than leaking the - * upstream client's raw session id. Same input → same UUID, so trace - * correlation across requests still works. - * - * Fires whenever `parseUserIdMetadata` produces a `sessionId`, regardless of - * whether the safety-identifier half is also present. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/api-config.ts (prepareInteractionHeaders, getRootSessionId) - * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/utils.ts#L217 (getRootSessionId) - * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/utils.ts#L230 (getUUID) - */ -const sessionUuid = async (input: string): Promise => { - const data = new TextEncoder().encode(input); - return v4({ random: new Uint8Array(await crypto.subtle.digest('SHA-256', data)) }); -}; - -export const withInteractionIdHeaderSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { - const { sessionId } = parseUserIdMetadata(ctx.payload.metadata?.user_id); - if (sessionId) { - ctx.headers.set('x-interaction-id', await sessionUuid(sessionId)); - } - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts deleted file mode 100644 index 6fb1800335..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { AnthropicMessagesAssistantMessage, AnthropicMessagesUserMessage } from '@floway-dev/protocols/anthropic-messages'; - -/** - * Copilot rejects Anthropic `image` blocks as plain text unless the private - * `copilot-vision-request: true` header is set. Detection must scan the final - * post-mutation payload (after other Anthropic Messages boundary interceptors have run) - * and cover both the top-level `message.content` and the nested - * `tool_result.content[]` shape; Anthropic allows images in both positions. - * - * Generic in the run-result type because the Copilot provider historically - * applied equivalent vision detection to every Anthropic Messages HTTP exchange (chat - * AND count_tokens). Keeping a single generic interceptor lets both the streaming - * Anthropic Messages boundary chain (`ExecuteResult<...>`) and the count_tokens chain - * (`Response`) share one definition. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/commit/1f6b98924ae092db9b2010846c32e5cbf10817df - */ -const contentHasImage = (content: AnthropicMessagesUserMessage['content'] | AnthropicMessagesAssistantMessage['content']): boolean => { - if (!Array.isArray(content)) return false; - return content.some(block => { - if (block.type === 'image') return true; - if (block.type === 'tool_result' && Array.isArray(block.content)) { - return block.content.some(inner => inner.type === 'image'); - } - return false; - }); -}; - -export const withVisionHeaderSet = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - if (ctx.payload.messages.some(message => contentHasImage(message.content))) { - ctx.headers.set('copilot-vision-request', 'true'); - } - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts deleted file mode 100644 index b4cdf58cda..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import { mapKeepingIdentity, withKeysChanged } from '@floway-dev/protocols/common'; - -/** - * Two `cache_control` sub-fields are beta extensions to the base - * `CacheControlEphemeral` shape that Copilot's stricter Anthropic-Messages-upstream - * deployments (claude-haiku-4.5, claude-sonnet-4.5/4.6, intermittently - * claude-opus-4.5) reject: - * - * - `scope`: added by Claude Code's `prompt-caching-scope-2025-11-27` beta. - * Copilot returns `cache_control.scope: Extra inputs are not permitted`. - * - `ttl`: added by the `extended-cache-ttl-2025-04-11` beta. Floway does - * not enable that beta for Copilot, so any `ttl` value trips the same - * schema rejection on the body. - * - * Walk every position where `cache_control` may appear — system blocks, - * tools, message content blocks including `tool_use` and `tool_result` — - * and strip both sub-fields, keeping `{ type: 'ephemeral' }` so prompt - * caching still primes on slots that do honour the marker. - * - * The top-level `cache_control` field is handled by - * `withTopLevelCacheControlApplied`, which runs first and either ports it - * onto the last cacheable block (where this interceptor cleans it) or - * deletes it when no cacheable block exists. - * - * Custom providers that speak Anthropic Messages directly may accept these - * betas natively; this interceptor is Copilot-only. - * - * References: - * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L2909-L2913 - * - https://github.com/caozhiyuan/copilot-api/issues/143 - * - https://github.com/caozhiyuan/copilot-api/issues/144 - * - https://github.com/caozhiyuan/copilot-api/issues/269 - * - https://github.com/caozhiyuan/copilot-api/commit/ce8224c55933f811abe5bf9ba42f9336a7852997 - */ -/** The block with its cache-control extensions gone, or the same block when it carried none. - * Returning the original is what lets an untouched message ride through by identity. */ -const stripExtensions = (block: T): T => { - const cacheControl = (block as Record).cache_control; - if (!cacheControl || typeof cacheControl !== 'object') return block; - - const { scope: _scope, ttl: _ttl, ...rest } = cacheControl as Record; - return withKeysChanged(block, { cache_control: Object.keys(rest).length > 0 ? rest : undefined }); -}; - -export const withCacheControlExtensionsStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { - const payload = ctx.payload; - - const system = Array.isArray(payload.system) - ? mapKeepingIdentity(payload.system, stripExtensions) - : payload.system; - const tools = payload.tools === undefined ? payload.tools : mapKeepingIdentity(payload.tools, stripExtensions); - const messages = mapKeepingIdentity(payload.messages, message => ( - Array.isArray(message.content) - // The content arrays are a union of per-role block types, so the walk is expressed over - // the shape they share — an object — and handed back at the array's own type. - ? withKeysChanged(message, { content: mapKeepingIdentity(message.content as readonly object[], stripExtensions) }) - : message - )); - - ctx.payload = withKeysChanged(payload, { - ...(system === payload.system ? {} : { system }), - ...(tools === payload.tools ? {} : { tools }), - ...(messages === payload.messages ? {} : { messages }), - }); - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts deleted file mode 100644 index e0ac90c66b..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; - -/** - * `eager_input_streaming` is a per-tool property in the Anthropic Messages API - * that enables fine-grained tool input streaming. Copilot's native Anthropic Messages - * target has been observed to reject it with - * `"tools.N.custom.eager_input_streaming: Extra inputs are not permitted"`, so - * strip it only at the Copilot target boundary and leave other providers - * untouched. - * - * References: - * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L1761 - */ -export const withEagerInputStreamingStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, run) => { - if (ctx.payload.tools) { - ctx.payload.tools = ctx.payload.tools.map(tool => { - const { eager_input_streaming: _, ...rest } = tool as typeof tool & { - eager_input_streaming?: unknown; - }; - return rest; - }); - } - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts deleted file mode 100644 index 8088f63f1d..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { Interceptor } from '@floway-dev/interceptor'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult, ProviderModel } from '@floway-dev/provider'; - -// Boundary ctx for Copilot Anthropic Messages interceptors. The chain runs inside -// `provider.callAnthropicMessages` after the gateway has handed control to the -// provider, so the gateway main flow no longer needs to know that Copilot -// has interceptors at all. -// -// `payload` is the source-shape body with `model` re-attached so interceptors -// that read the public model id (e.g. claude-opus-4-8 carve-outs) keep -// working unchanged; the terminal strips it before serializing to the wire. -// `headers` carries the provider's ordinary admitted headers. -// Parsed `anthropic-beta` tokens arrive through the typed Anthropic Messages call -// contract and are copied into `anthropicBeta` so variant selection and -// interceptors share one token list. The terminal serializes the normalized -// list back onto a fresh wire-header bag. -export interface AnthropicMessagesBoundaryCtx { - payload: AnthropicMessagesPayload; - headers: Headers; - anthropicBeta: string[]; - readonly model: ProviderModel; -} - -export type CopilotAnthropicMessagesBoundaryInterceptor = Interceptor< - AnthropicMessagesBoundaryCtx, - ExecuteResult> ->; - -// count_tokens is a one-shot, non-streaming HTTP exchange: the terminal -// returns the raw upstream `Response` directly. Pure header/payload mutators -// only — post-`run()` event-stream inspection is not portable to this result. -export type CopilotAnthropicMessagesCountTokensBoundaryInterceptor = Interceptor< - AnthropicMessagesBoundaryCtx, - Response ->; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts deleted file mode 100644 index abe74ea8c7..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; -import { checkWhitespaceOverflow } from '../shared/whitespace-overflow.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; - -/** - * Copilot has been observed to emit only whitespace (`\r`, `\n`, `\t`) inside - * `function.arguments` deltas until `max_tokens`, never producing valid JSON. - * Detect that pattern per tool call and abort by throwing, so every source - * (native OpenAI Chat Completions, plus Anthropic Messages/Gemini generateContent/OpenAI Responses - * sources translated via OpenAI Chat Completions) - * sees the gateway's standard upstream-error path. - * - * The OpenAI Chat Completions protocol cannot express a stream error in-band: the - * `finish_reason` enum lacks an 'error' value, and the de-facto - * `data: {"error":{...}}` chunk pattern is only recognized by some - * translators (e.g. gemini-generate-content-via-openai-chat-completions calls - * `openaiChatCompletionsErrorPayloadMessage`) but not others - * (e.g. anthropic-messages-via-openai-chat-completions iterates `choices[].delta` and would - * silently drop an error chunk). Throwing keeps the abort semantics uniform - * across every consumer; the source layer converts the thrown error into the - * downstream protocol's native error event. - * - * Lives at the Copilot provider boundary so other OpenAI-compatible providers - * are not slowed by per-delta whitespace inspection. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/commit/4c0d775e1dc6b8648c7ad5f21fb783fc3246facf - * - https://github.com/caozhiyuan/copilot-api/commit/3cdc32c0811469da9eebec5ca3892caf068df542 - */ -const isWhitespaceExceeded = ( - chunk: OpenAIChatCompletionsStreamEvent, - whitespaceByIndex: Map, -): boolean => { - for (const choice of chunk.choices) { - const toolCalls = choice.delta.tool_calls; - if (!toolCalls) continue; - - for (const toolCall of toolCalls) { - const args = toolCall.function?.arguments; - if (!args) continue; - - const current = whitespaceByIndex.get(toolCall.index) ?? 0; - const { count, exceeded } = checkWhitespaceOverflow(args, current); - whitespaceByIndex.set(toolCall.index, count); - if (exceeded) return true; - } - } - return false; -}; - -export const withToolArgumentWhitespaceAborted: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (_invocation, run) => { - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* (): AsyncGenerator> { - const whitespaceByIndex = new Map(); - - for await (const frame of result.events) { - if (frame.type === 'event' && isWhitespaceExceeded(frame.event, whitespaceByIndex)) { - throw new Error('Copilot tool call arguments contained excessive consecutive whitespace, indicating a degenerate response.'); - } - yield frame; - } - })(), - }; -}; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts deleted file mode 100644 index d0a29d4522..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts +++ /dev/null @@ -1,82 +0,0 @@ -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; -import type { OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; - -/** - * Prime Copilot's content-addressed prompt cache by tagging a small set of - * messages with `copilot_cache_control: { type: 'ephemeral' }`. Selection: - * the first two eligible system messages and the last two eligible - * non-system messages, in original order. - * - * The field is a Copilot-private extension to the OpenAI Chat Completions - * wire shape — kept off the shared `OpenAIChatCompletionsMessage` type so other OpenAI-compatible - * upstreams never see it on their inbound payloads, and attached here through - * a Copilot-local view that widens `OpenAIChatCompletionsMessage` with the marker. - * - * Probe findings (not obvious from the upstream reference): - * - * 1. Cache hits surface ONLY via `usage.prompt_tokens_details.cached_tokens` - * in Copilot's responses. There is no `cache_creation_input_tokens` on - * Copilot regardless of model family. - * 2. The markers are needed only to PRIME the cache. Once a prompt prefix has - * been primed, subsequent calls hitting the same content (byte-for-byte) - * will get a cache hit even when no `copilot_cache_control` is attached. - * The cache is content-addressed, not marker-addressed. - * 3. Copilot internally decides whether to actually cache based on prompt - * size (~1.7k tokens observed as the lower bound). We always attach so - * short prompts pay nothing and long ones get primed automatically. - * 4. The marker is silently accepted by every observed model family - * (gpt-5.4, gpt-5-mini, gpt-4o-mini, claude-sonnet-4.5). No 4xx, no - * schema rejection, no behavioral difference beyond cache priming. - * - * Primes every source path routed through Copilot OpenAI Chat Completions — native - * OpenAI Chat Completions, Anthropic-Messages-via-OpenAI-Chat-Completions, OpenAI-Responses-via-OpenAI-Chat-Completions, and Gemini-generateContent-via-OpenAI-Chat-Completions. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/9be0eb602f1ffee7597741c9af9bc66a68e1a241/src/routes/messages/api-flows.ts#L381-L432 - */ - -export interface CopilotCacheableMessage extends OpenAIChatCompletionsMessage { - copilot_cache_control?: { type: 'ephemeral' }; -} - -const COPILOT_CONTEXT_CACHE_SYSTEM_MARKER_LIMIT = 2; -const COPILOT_CONTEXT_CACHE_NON_SYSTEM_MARKER_LIMIT = 2; -const COPILOT_CONTEXT_CACHE_CONTROL = { type: 'ephemeral' } as const; - -const isEligible = (message: OpenAIChatCompletionsMessage): boolean => { - const { content } = message; - if (typeof content === 'string') return content.length > 0; - return Array.isArray(content) && content.length > 0; -}; - -const selectCacheMarkerIndexes = (messages: readonly OpenAIChatCompletionsMessage[]): number[] => { - const systemIndexes: number[] = []; - for (let i = 0; i < messages.length && systemIndexes.length < COPILOT_CONTEXT_CACHE_SYSTEM_MARKER_LIMIT; i++) { - if (messages[i].role === 'system' && isEligible(messages[i])) systemIndexes.push(i); - } - - const nonSystemIndexes: number[] = []; - for (let i = messages.length - 1; i >= 0 && nonSystemIndexes.length < COPILOT_CONTEXT_CACHE_NON_SYSTEM_MARKER_LIMIT; i--) { - if (messages[i].role !== 'system' && isEligible(messages[i])) nonSystemIndexes.push(i); - } - - return [...new Set([...systemIndexes, ...nonSystemIndexes])].sort((a, b) => a - b); -}; - -export const withCacheControlMarkersAttached: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, run) => { - const marked = new Set(selectCacheMarkerIndexes(ctx.payload.messages)); - if (marked.size === 0) return await run(); - - ctx.payload = { - ...ctx.payload, - // A marker is its own object per message, so nothing downstream can carry a change from - // one to another. The messages that were not selected ride through by identity. - messages: ctx.payload.messages.map((message, index) => ( - marked.has(index) - ? { ...message, copilot_cache_control: { ...COPILOT_CONTEXT_CACHE_CONTROL } } as CopilotCacheableMessage - : message - )), - }; - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts deleted file mode 100644 index 4f6cd05217..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { memoizedDataUrlCompressor } from '../image-compression.ts'; -import { targetSizeForOpenAIResponsesChat } from '../image-size.ts'; -import type { OpenAIChatCompletionsBoundaryCtx, CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; -import type { OpenAIChatCompletionsContentPart, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; -import { isBase64ImageDataUrl } from '@floway-dev/provider'; - -type OpenAIChatCompletionsImagePart = Extract; - -const compressInlineImages = async (ctx: OpenAIChatCompletionsBoundaryCtx): Promise => { - const targets: OpenAIChatCompletionsImagePart[] = []; - for (const message of ctx.payload.messages) { - if (!Array.isArray(message.content)) continue; - for (const part of message.content) { - if (part.type === 'image_url' && isBase64ImageDataUrl(part.image_url.url)) targets.push(part); - } - } - - if (targets.length === 0) return; - - const compress = memoizedDataUrlCompressor(targetSizeForOpenAIResponsesChat(ctx.model.id)); - const compressedUrls = new Map(); - await Promise.all( - targets.map(async target => { - compressedUrls.set(target, await compress(target.image_url.url)); - }), - ); - const hasCompressedImage = (part: OpenAIChatCompletionsContentPart): part is OpenAIChatCompletionsImagePart => - part.type === 'image_url' && compressedUrls.has(part); - const rewriteImage = (part: OpenAIChatCompletionsImagePart): OpenAIChatCompletionsImagePart => { - const url = compressedUrls.get(part); - if (url === undefined) throw new Error('Missing compressed OpenAI Chat Completions image URL'); - return { ...part, image_url: { ...part.image_url, url } }; - }; - ctx.payload = { - ...ctx.payload, - messages: ctx.payload.messages.map((message): OpenAIChatCompletionsMessage => { - if (!Array.isArray(message.content) || !message.content.some(hasCompressedImage)) return message; - return { - ...message, - content: message.content.map(part => hasCompressedImage(part) ? rewriteImage(part) : part), - }; - }), - }; -}; - -// Recompresses every inline base64 image (`data:image/*;base64,...` in an -// `image_url` part) in the outgoing OpenAI Chat Completions payload to WebP before -// the Copilot upstream call. Remote https image references are left untouched. -export const withInlineImagesCompressed: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, run) => { - // Finish this nested activation before starting the upstream call. Its - // request-local memoizer keys are the full source data URLs, which can be - // several megabytes each and must not stay live for the response stream. - await compressInlineImages(ctx); - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/index.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/index.ts deleted file mode 100644 index 586e019a8b..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -// Copilot-only OpenAI Chat Completions workarounds. The boundary chain runs inside -// `provider.callOpenAIChatCompletions`, so the gateway main flow never knows that -// Copilot has OpenAI Chat Completions interceptors at all. - -import { withToolArgumentWhitespaceAborted } from './abort-on-tool-argument-whitespace.ts'; -import { withCacheControlMarkersAttached } from './attach-cache-control-markers.ts'; -import { withInlineImagesCompressed } from './compress-images.ts'; -import { withInitiatorHeaderSet } from './set-initiator-header.ts'; -import { withVisionHeaderSet } from './set-vision-header.ts'; -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; - -// Order matters: payload-mutating interceptors run first so the header -// interceptors see the final outgoing payload, then the header interceptors -// populate `ctx.headers` for the upstream call. Cache-control marker -// attachment is a payload mutator, so it sits with the other payload mutators -// and before any header derivation. -export const COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY = [ - withInlineImagesCompressed, - withToolArgumentWhitespaceAborted, - withCacheControlMarkersAttached, - withInitiatorHeaderSet, - withVisionHeaderSet, -] as const satisfies readonly CopilotOpenAIChatCompletionsBoundaryInterceptor[]; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts deleted file mode 100644 index 917ef12042..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; - -/** - * Copilot's `x-initiator` header distinguishes user-triggered turns from - * agent-triggered tool-result consumption. On OpenAI Chat Completions the - * discriminator is the last message: when its role is `assistant` (model - * replay) or `tool` (a tool result being fed back into the model), the agent - * is driving the turn. - * - * OpenAI Responses tool-output images expose a deliberate translation loss here: - * OpenAI Chat Completions tool messages cannot carry image parts, so translation lifts them into - * a legal user message after the contiguous tool results. The OpenAI Chat Completions wire role - * remains authoritative, and that turn is therefore reported as user-initiated - * even though its image originated in tool output. Preserving the source-side - * provenance would require a contradictory out-of-band signal. - * - * Official clients do not offer one wire-derived rule to copy. VS Code keeps - * image content on a private multimodal `role: "tool"` OpenAI Chat Completions message and lifts - * only for OpenAI Responses, while initiator provenance can travel separately from - * serialized roles. Floway has neither private OpenAI Chat Completions tool-image syntax nor - * client loop metadata; this remains a payload-shape heuristic, and translated - * or synthetic final user messages can therefore be classified differently - * from their source-side turn. - * - * The header name is lowercase `x-initiator`; HTTP header names are - * case-insensitive on the wire, so the casing is cosmetic. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/services/copilot/create-chat-completions.ts#L28-L49 - * - https://github.com/openai/openai-node/blob/61539248cbe04665de68a71e6fd878127ae4db87/src/resources/chat/completions/completions.ts#L1893-L1908 - * - https://github.com/microsoft/vscode-prompt-tsx/blob/86cc3a025fb54b72b0b5be9ddbc786ea16ef4073/src/base/output/openaiConvert.ts#L15-L81 - * - https://github.com/microsoft/vscode/blob/fb5e582d1c8edb9ad0a69e50fe6f508a8c095466/extensions/copilot/src/platform/endpoint/node/responsesApi.ts#L419-L468 - * - https://github.com/microsoft/vscode/blob/fb5e582d1c8edb9ad0a69e50fe6f508a8c095466/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts#L1453-L1474 - */ -export const withInitiatorHeaderSet: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, run) => { - const lastMessage = ctx.payload.messages.at(-1); - const agentInitiated = lastMessage?.role === 'assistant' - || lastMessage?.role === 'tool'; - ctx.headers.set('x-initiator', agentInitiated ? 'agent' : 'user'); - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts deleted file mode 100644 index 0e932d2ad3..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; - -/** - * Copilot's OpenAI Chat Completions endpoint requires the private - * `copilot-vision-request: true` header before it accepts OpenAI-style - * `image_url` content parts; without it images are silently dropped or - * rejected. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/services/copilot/create-chat-completions.ts#L28-L49 - */ -export const withVisionHeaderSet: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, run) => { - const hasImage = ctx.payload.messages.some( - message => Array.isArray(message.content) && message.content.some(part => part.type === 'image_url'), - ); - if (hasImage) ctx.headers.set('copilot-vision-request', 'true'); - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts deleted file mode 100644 index 0c3d70700c..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult, ProviderModel } from '@floway-dev/provider'; - -// Boundary ctx for Copilot OpenAI Chat Completions interceptors. See anthropic-messages/types.ts -// for the boundary-isolation rationale. -export interface OpenAIChatCompletionsBoundaryCtx { - payload: OpenAIChatCompletionsPayload; - headers: Headers; - readonly model: ProviderModel; -} - -export type CopilotOpenAIChatCompletionsBoundaryInterceptor = Interceptor< - OpenAIChatCompletionsBoundaryCtx, - ExecuteResult> ->; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts deleted file mode 100644 index 76b051110f..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { CopilotOpenAIResponsesBoundaryInterceptor } from './types.ts'; -import { checkWhitespaceOverflow } from '../shared/whitespace-overflow.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; - -/** - * Copilot has been observed to emit only whitespace (`\r`, `\n`, `\t`) inside - * `response.function_call_arguments.delta` events until `max_tokens`, never - * producing valid JSON arguments. Detect that pattern per function call output - * index and abort the upstream stream before the client times out. - * - * Behaviour: when any single output index's argument deltas accumulate more - * than `MAX_CONSECUTIVE_WHITESPACE` consecutive whitespace characters, emit a - * OpenAI Responses `error` event followed by a done frame, then end the stream. - * The OpenAI Responses source layer surfaces `error` events as a stream failure. - * - * Lives at the Copilot provider boundary so other OpenAI-Responses-capable providers - * are not slowed by per-delta whitespace inspection. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/commit/4c0d775e1dc6b8648c7ad5f21fb783fc3246facf - * - https://github.com/caozhiyuan/copilot-api/commit/3cdc32c0811469da9eebec5ca3892caf068df542 - */ -const ABORT_MESSAGE = 'Tool call arguments contained excessive whitespace, indicating a degenerate response.'; - -const isArgumentsDelta = (event: OpenAIResponsesStreamEvent): event is OpenAIResponsesStreamEvent & { type: 'response.function_call_arguments.delta'; output_index: number; delta: string } => - event.type === 'response.function_call_arguments.delta'; - -const errorEvent = (): OpenAIResponsesStreamEvent => - ({ - type: 'error', - message: ABORT_MESSAGE, - code: 'api_error', - }) as OpenAIResponsesStreamEvent; - -export const withToolArgumentWhitespaceAborted: CopilotOpenAIResponsesBoundaryInterceptor = async (_invocation, run) => { - const result = await run(); - // Only the streaming generate branch produces events worth inspecting. - // The compact branch is a single value envelope; pass it through unchanged. - if (result.action !== 'generate' || !result.ok) return result; - - return { - ...result, - events: (async function* (): AsyncGenerator> { - const whitespaceByIndex = new Map(); - - for await (const frame of result.events) { - if (frame.type !== 'event' || !isArgumentsDelta(frame.event)) { - yield frame; - continue; - } - - const event = frame.event; - const current = whitespaceByIndex.get(event.output_index) ?? 0; - const { count, exceeded } = checkWhitespaceOverflow(event.delta, current); - whitespaceByIndex.set(event.output_index, count); - - if (exceeded) { - console.warn('Copilot: infinite whitespace detected in OpenAI Responses function call arguments, aborting stream'); - yield eventFrame(errorEvent()); - yield doneFrame(); - return; - } - - yield frame; - } - })(), - }; -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts b/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts deleted file mode 100644 index 1c4d0b8dd3..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { memoizedDataUrlCompressor } from '../image-compression.ts'; -import { targetSizeForOpenAIResponsesChat } from '../image-size.ts'; -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import type { OpenAIResponsesInputContent, OpenAIResponsesInputImage, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; -import { isBase64ImageDataUrl } from '@floway-dev/provider'; - -// A single client request can reach this boundary more than once — a -// fallback candidate attempt, or another turn of the OpenAI Responses server-tool -// shim's loop — carrying the same nested image part. Remember the exact -// generated URL on that request-owned object so the repeat neither -// re-encodes a lossy WebP nor mistakes an unrelated client WebP for our -// output. The non-enumerable property stays off the wire and does not -// cross an object-spread/JSON ownership boundary. -const compressedImageUrl = Symbol('compressedImageUrl'); -type CompressibleImagePart = OpenAIResponsesInputImage & { image_url: string; [compressedImageUrl]?: string }; - -const compressInlineImages = async (ctx: OpenAIResponsesBoundaryCtx): Promise => { - const targets: Array<{ part: CompressibleImagePart; imageUrl: string }> = []; - for (const item of ctx.payload.input) { - const parts = item.type === 'message' - ? item.content - : item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.output : undefined; - if (!Array.isArray(parts)) continue; - for (const part of parts) { - if (part.type !== 'input_image' || typeof part.image_url !== 'string') continue; - const imagePart = part as CompressibleImagePart; - if (imagePart[compressedImageUrl] === imagePart.image_url || !isBase64ImageDataUrl(imagePart.image_url)) continue; - targets.push({ part: imagePart, imageUrl: imagePart.image_url }); - } - } - - if (targets.length === 0) return; - - const compress = memoizedDataUrlCompressor(targetSizeForOpenAIResponsesChat(ctx.model.id)); - const compressedUrls = new Map(); - await Promise.all( - targets.map(async target => { - compressedUrls.set(target.part, await compress(target.imageUrl)); - }), - ); - const hasCompressedImage = (part: OpenAIResponsesInputContent): part is CompressibleImagePart => - part.type === 'input_image' && compressedUrls.has(part as CompressibleImagePart); - const rewriteImage = (part: CompressibleImagePart): CompressibleImagePart => { - const imageUrl = compressedUrls.get(part); - if (imageUrl === undefined) throw new Error('Missing compressed OpenAI Responses image URL'); - const rewritten: CompressibleImagePart = { ...part, image_url: imageUrl }; - Object.defineProperty(rewritten, compressedImageUrl, { - configurable: true, - value: imageUrl, - writable: true, - }); - return rewritten; - }; - const rewriteParts = (parts: TPart[]): TPart[] => - parts.map(part => hasCompressedImage(part) ? rewriteImage(part) as TPart : part); - - ctx.payload = { - ...ctx.payload, - input: ctx.payload.input.map(item => { - if (item.type === 'message' && Array.isArray(item.content)) { - return item.content.some(hasCompressedImage) - ? { ...item, content: rewriteParts(item.content) } - : item; - } - if ((item.type === 'function_call_output' || item.type === 'custom_tool_call_output') && Array.isArray(item.output)) { - return item.output.some(hasCompressedImage) - ? { ...item, output: rewriteParts(item.output) } - : item; - } - return item; - }), - }; -}; - -// Recompresses every inline base64 image in the outgoing OpenAI Responses payload to -// WebP before the Copilot upstream call. Images appear both as `input_image` -// parts inside message content and inside function/custom tool outputs -// (multimodal tool results, e.g. a screenshot tool). Remote https and file-id -// references are left untouched. Generic in the run-result type so the same -// definition feeds both the streaming `/responses` chain and the non-streaming -// compaction chain. -export const withInlineImagesCompressed = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - // Finish this nested activation before starting the upstream call. Its - // request-local memoizer keys are the full source data URLs, which can be - // several megabytes each and must not stay live for the response stream. - await compressInlineImages(ctx); - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts b/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts deleted file mode 100644 index 08c6949c1e..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import { mapOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses'; - -/** - * Some Copilot data planes reject empty namespace descriptions permitted by - * the current OpenAI schema. Fill the permitted-empty text at the Copilot - * boundary so the same request remains portable across those deployments. - * - * References: - * - https://github.com/openai/openai-openapi/commit/466c74a42f51c02f1927bc666815251dc53845dc - * - https://github.com/caozhiyuan/copilot-api/issues/345 - * - https://github.com/engineersamuel/copilot-proxy-rs/commit/1c23d7237bd1bd7c24045d0d426412851bfd57f6 - */ -export const withEmptyNamespaceDescriptionsFilled = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = mapOpenAIResponsesTools(ctx.payload, tool => tool.type === 'namespace' && tool.description === '' - ? { ...tool, description: `Tools in the ${tool.name} namespace.` } - : tool); - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts b/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts deleted file mode 100644 index efcb103d6c..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; - -/** - * Copilot's `/responses` rejects `store: true` with - * `400 {"error":{"message":"store is not supported","code":"unsupported_value","param":"store"}}`. - * Force `store: false` on the outgoing payload once planning has committed to - * the Copilot OpenAI Responses target so the upstream accepts the request. The - * gateway's own stored-items persistence keys off the caller's original `store` - * value captured at parse time and is unaffected by this upstream-only flag. - * - * Generic in the run-result type so the same definition feeds both the - * streaming `/responses` chain and the non-streaming compaction chain — the - * synth-via-trigger compact call also rejects `store: true`. - */ -export const withStoreForcedFalse = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = { ...ctx.payload, store: false }; - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/index.ts b/packages/provider-copilot/src/interceptors/openai-responses/index.ts deleted file mode 100644 index b65f9b03f0..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Copilot-only OpenAI Responses workarounds. The chain is a boundary the Copilot -// provider runs inside its own `callX` methods, so the gateway main flow -// never knows that Copilot has OpenAI Responses interceptors at all. - -import { withToolArgumentWhitespaceAborted } from './abort-on-tool-argument-whitespace.ts'; -import { withInlineImagesCompressed } from './compress-images.ts'; -import { withEmptyNamespaceDescriptionsFilled } from './fill-empty-namespace-descriptions.ts'; -import { withStoreForcedFalse } from './force-store-false.ts'; -import { withCopilotOpenAIResponsesItemIdMembrane } from './item-id-membrane.ts'; -import { withInitiatorHeaderSet } from './set-initiator-header.ts'; -import { withVisionHeaderSet } from './set-vision-header.ts'; -import { withImageGenerationStripped } from './strip-image-generation.ts'; -import { withServiceTierStripped } from './strip-service-tier.ts'; -import type { CopilotOpenAIResponsesBoundaryInterceptor } from './types.ts'; - -// Single chain wraps both the streaming `/responses` call and the -// non-streaming synth-via-trigger compaction call — the chain terminal -// switches on `ctx.action` to pick the wire shape. Order matters: -// payload-mutating interceptors run first so the header interceptors see -// the final outgoing payload, then the header interceptors populate -// `ctx.headers` for the upstream call. Result mutators sit between: the -// whitespace guard acts only on generate streams, while the item-id membrane -// also normalizes the generated item in compact value envelopes. -export const COPILOT_OPENAI_RESPONSES_BOUNDARY = [ - withInlineImagesCompressed, - withEmptyNamespaceDescriptionsFilled, - withServiceTierStripped, - withImageGenerationStripped, - withStoreForcedFalse, - withCopilotOpenAIResponsesItemIdMembrane, - withToolArgumentWhitespaceAborted, - withVisionHeaderSet, - withInitiatorHeaderSet, -] as const satisfies readonly CopilotOpenAIResponsesBoundaryInterceptor[]; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts b/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts deleted file mode 100644 index 8f3a82e1bf..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts +++ /dev/null @@ -1,279 +0,0 @@ -import { unwrapCopilotItemId, wrapCopilotItemId } from './item-id-carrier.ts'; -import type { CopilotOpenAIResponsesBoundaryInterceptor } from './types.ts'; -import { encodeHex, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; - -// OpenAI's published examples establish these item-specific prefixes. Keeping -// the Copilot output inventory closed prevents a new upstream item kind from -// leaking its raw id before we have verified its replay behavior. -// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L57042-L59599 -// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L68023-L68281 -// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L68333-L68748 -// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L74970-L75020 -const COPILOT_OUTPUT_ITEM_POLICIES = { - message: { prefix: 'msg', carrier: null }, - reasoning: { prefix: 'rs', carrier: 'encrypted_content' }, - function_call: { prefix: 'fc', carrier: null }, - custom_tool_call: { prefix: 'ctc', carrier: null }, - web_search_call: { prefix: 'ws', carrier: null }, - tool_search_call: { prefix: 'tsc', carrier: null }, - tool_search_output: { prefix: 'tso', carrier: null }, - program: { prefix: 'cm', carrier: 'fingerprint' }, - program_output: { prefix: 'cmo', carrier: null }, - agent_message: { prefix: 'amsg', carrier: 'agent_content' }, - compaction: { prefix: 'cmp', carrier: 'encrypted_content' }, - shell_call: { prefix: 'sh', carrier: null }, - shell_call_output: { prefix: 'sho', carrier: null }, - apply_patch_call: { prefix: 'apc', carrier: null }, -} as const; - -type CopilotOutputItemType = keyof typeof COPILOT_OUTPUT_ITEM_POLICIES; -type CarrierItem = OpenAIResponsesInputItem | OpenAIResponsesOutputItem; - -const copilotOutputItemType = (item: OpenAIResponsesOutputItem): CopilotOutputItemType => { - if (Object.hasOwn(COPILOT_OUTPUT_ITEM_POLICIES, item.type)) return item.type as CopilotOutputItemType; - throw new TypeError(`Unsupported Copilot OpenAI Responses output item type '${item.type}'`); -}; - -const createPublicItemId = (type: CopilotOutputItemType): string => { - const bytes = crypto.getRandomValues(new Uint8Array(16)); - const suffix = encodeHex(bytes); - return `${COPILOT_OUTPUT_ITEM_POLICIES[type].prefix}_${suffix}`; -}; - -const mapCarrierValues = ( - item: TItem, - transform: (value: string) => string, -): TItem => { - if (!Object.hasOwn(COPILOT_OUTPUT_ITEM_POLICIES, item.type)) return item; - const policy = COPILOT_OUTPUT_ITEM_POLICIES[item.type as CopilotOutputItemType]; - switch (policy.carrier) { - case 'encrypted_content': { - const record = item as CarrierItem & { encrypted_content?: unknown }; - return typeof record.encrypted_content === 'string' - ? { ...item, encrypted_content: transform(record.encrypted_content) } as TItem - : item; - } - case 'fingerprint': { - const record = item as CarrierItem & { fingerprint: string }; - return { ...item, fingerprint: transform(record.fingerprint) } as TItem; - } - case 'agent_content': { - const record = item as Extract; - return { - ...item, - content: record.content.map(content => - content.type === 'encrypted_content' && typeof content.encrypted_content === 'string' - ? { ...content, encrypted_content: transform(content.encrypted_content) } - : content), - } as TItem; - } - case null: - return item; - } -}; - -const restoreInputItem = (item: OpenAIResponsesInputItem): OpenAIResponsesInputItem => { - const upstreamItemIds = new Set(); - const restored = mapCarrierValues(item, value => { - const decoded = unwrapCopilotItemId(value); - if (decoded.kind === 'foreign') return value; - upstreamItemIds.add(decoded.id); - return decoded.value; - }); - - if (upstreamItemIds.size === 0) return restored; - if (upstreamItemIds.size > 1) { - throw new TypeError('Copilot OpenAI Responses item carries conflicting upstream ids'); - } - return { ...restored, id: [...upstreamItemIds][0] } as OpenAIResponsesInputItem; -}; - -const restoreInputItemIds = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => ({ - ...payload, - input: payload.input.map(restoreInputItem), -}); - -const carrierValueCount = (item: OpenAIResponsesOutputItem): number => { - let count = 0; - mapCarrierValues(item, value => { - count += 1; - return value; - }); - return count; -}; - -const normalizeObservedItem = (item: OpenAIResponsesOutputItem, publicId: string): OpenAIResponsesOutputItem => { - copilotOutputItemType(item); - if (carrierValueCount(item) === 0) return { ...item, id: publicId } as OpenAIResponsesOutputItem; - - const upstreamId = 'id' in item ? item.id : undefined; - if (typeof upstreamId !== 'string' || upstreamId.length === 0) { - throw new TypeError(`Copilot OpenAI Responses ${item.type} item has replay state but no upstream id`); - } - return { - ...mapCarrierValues(item, value => wrapCopilotItemId(value, upstreamId)), - id: publicId, - } as OpenAIResponsesOutputItem; -}; - -interface TrackedItem { - readonly type: CopilotOutputItemType; - readonly publicId: string; - added: boolean; -} - -interface StreamItemState { - readonly items: Map; -} - -const trackedAt = (state: StreamItemState, outputIndex: number): TrackedItem => { - const tracked = state.items.get(outputIndex); - if (tracked === undefined) throw new TypeError(`Copilot OpenAI Responses event references output_index ${outputIndex} before output_item.added`); - return tracked; -}; - -const trackObservedItem = ( - state: StreamItemState, - outputIndex: number, - item: OpenAIResponsesOutputItem, -): TrackedItem => { - const type = copilotOutputItemType(item); - const existing = state.items.get(outputIndex); - if (existing === undefined) { - const tracked: TrackedItem = { type, publicId: createPublicItemId(type), added: false }; - state.items.set(outputIndex, tracked); - return tracked; - } - if (existing.type !== type) { - throw new TypeError(`Copilot OpenAI Responses output_index ${outputIndex} changed type from ${existing.type} to ${item.type}`); - } - return existing; -}; - -const normalizeResponseOutput = ( - response: OpenAIResponsesResult, - state: StreamItemState, -): OpenAIResponsesResult => { - if (response.output.length === 0) return response; - return { - ...response, - output: response.output.map((item, outputIndex) => { - const tracked = trackObservedItem(state, outputIndex, item); - return normalizeObservedItem(item, tracked.publicId); - }), - }; -}; - -const ITEM_ID_EVENT_TYPES = new Set([ - 'response.content_part.added', - 'response.content_part.done', - 'response.reasoning_summary_part.added', - 'response.reasoning_summary_part.done', - 'response.reasoning_summary_text.delta', - 'response.reasoning_summary_text.done', - 'response.reasoning_text.delta', - 'response.reasoning_text.done', - 'response.output_text.delta', - 'response.output_text.done', - 'response.output_text.annotation.added', - 'response.web_search_call.in_progress', - 'response.web_search_call.searching', - 'response.web_search_call.completed', - 'response.image_generation_call.in_progress', - 'response.image_generation_call.generating', - 'response.image_generation_call.partial_image', - 'response.image_generation_call.completed', - 'response.function_call_arguments.delta', - 'response.function_call_arguments.done', - 'response.custom_tool_call_input.delta', - 'response.custom_tool_call_input.done', - 'response.apply_patch_call_operation_diff.delta', - 'response.apply_patch_call_operation_diff.done', -]); - -const NO_ITEM_ID_EVENT_TYPES = new Set([ - 'response.shell_call_command.added', - 'response.shell_call_command.delta', - 'response.shell_call_command.done', -]); - -const normalizeStreamEvent = (event: OpenAIResponsesStreamEvent, state: StreamItemState): OpenAIResponsesStreamEvent => { - if (event.type === 'response.output_item.added') { - const tracked = trackObservedItem(state, event.output_index, event.item); - if (tracked.added) { - throw new TypeError(`Copilot OpenAI Responses emitted output_item.added twice for output_index ${event.output_index}`); - } - tracked.added = true; - return { ...event, item: normalizeObservedItem(event.item, tracked.publicId) }; - } - - if (event.type === 'response.output_item.done') { - const tracked = trackObservedItem(state, event.output_index, event.item); - return { ...event, item: normalizeObservedItem(event.item, tracked.publicId) }; - } - - if ( - event.type === 'response.queued' - || event.type === 'response.created' - || event.type === 'response.in_progress' - || event.type === 'response.completed' - || event.type === 'response.incomplete' - || event.type === 'response.failed' - ) { - return { ...event, response: normalizeResponseOutput(event.response, state) }; - } - - if (event.type === 'error') return event; - const carrier = event as OpenAIResponsesStreamEvent & { item_id?: unknown; output_index?: unknown }; - const requiresItemId = ITEM_ID_EVENT_TYPES.has(event.type); - const permitsMissingItemId = NO_ITEM_ID_EVENT_TYPES.has(event.type); - if (!requiresItemId && !permitsMissingItemId) { - if (Object.hasOwn(event, 'item') || Object.hasOwn(event, 'response')) { - throw new TypeError(`Unsupported Copilot OpenAI Responses stream event type '${event.type}'`); - } - if (!Object.hasOwn(carrier, 'item_id')) return event; - } else if (permitsMissingItemId && !Object.hasOwn(carrier, 'item_id')) { - return event; - } - if (typeof carrier.item_id !== 'string') { - const reason = requiresItemId ? 'is missing item_id' : 'carries an invalid item_id extension'; - throw new TypeError(`Copilot OpenAI Responses event '${event.type}' ${reason}`); - } - if (typeof carrier.output_index !== 'number') { - if (!requiresItemId) { - throw new TypeError(`Copilot OpenAI Responses event '${event.type}' carries an invalid item_id extension`); - } - throw new TypeError(`Copilot OpenAI Responses event '${event.type}' carries item_id without output_index`); - } - return { ...carrier, item_id: trackedAt(state, carrier.output_index).publicId } as OpenAIResponsesStreamEvent; -}; - -const normalizeFrames = async function* ( - frames: AsyncIterable>, -): AsyncGenerator> { - const state: StreamItemState = { items: new Map() }; - for await (const frame of frames) { - yield frame.type === 'event' - ? { ...frame, event: normalizeStreamEvent(frame.event, state) } - : frame; - } -}; - -const normalizeCompactionResult = (response: OpenAIResponsesCompactionResult): OpenAIResponsesCompactionResult => ({ - ...response, - output: response.output.map(item => { - if (item.type !== 'compaction') return item; - return normalizeObservedItem(item, createPublicItemId('compaction')); - }), -}); - -export const withCopilotOpenAIResponsesItemIdMembrane: CopilotOpenAIResponsesBoundaryInterceptor = async (ctx, run) => { - ctx.payload = restoreInputItemIds(ctx.payload); - const result = await run(); - if (!result.ok) return result; - - return result.action === 'generate' - ? { ...result, events: normalizeFrames(result.events) } - : { ...result, result: normalizeCompactionResult(result.result) }; -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts b/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts deleted file mode 100644 index 5806b3e356..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; - -/** - * Copilot's `x-initiator` header distinguishes user-triggered turns from - * agent-triggered tool-result consumption. On OpenAI Responses the discriminator is - * the last input item — but the set of "agent" shapes is broader than just - * `function_call_output`: - * - * - Tool-output-style items lack a `role` field entirely - * (`function_call_output`, `custom_tool_call_output`, `tool_search_output`, - * plus any future hosted-tool output shape). Classify them as agent. - * - An assistant message replayed back into `input` is also agent-driven. - * - * Role-bearing user / system / developer items — including the canonical - * `additional_tools` developer item — and an empty input mean initiator = user. - * - * The header name is lowercase `x-initiator`; HTTP header names are - * case-insensitive on the wire, so the casing is cosmetic. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/cd8207cb70ede07771bf37a04accfbf2af76d980/src/routes/responses/utils.ts#L75-L87 - * (`hasAgentInitiator`) - */ -export const withInitiatorHeaderSet = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - const lastItem: OpenAIResponsesInputItem | undefined = ctx.payload.input.at(-1); - const role = lastItem === undefined ? undefined : (lastItem as { role?: unknown }).role; - const initiator: 'user' | 'agent' = lastItem !== undefined - && (role === undefined || role === null || role === '' || role === 'assistant') - ? 'agent' - : 'user'; - ctx.headers.set('x-initiator', initiator); - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts b/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts deleted file mode 100644 index d876f505cf..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; - -/** - * Copilot's OpenAI Responses endpoint requires the private - * `copilot-vision-request: true` header to accept image inputs. Canonical - * OpenAI Responses carries `input_image` blocks in message content and in multimodal - * function/custom tool output arrays, so all three containers participate in - * the same detection rule. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/lib/api-config.ts#L248-L258 - * - https://github.com/caozhiyuan/copilot-api/blob/cd8207cb70ede07771bf37a04accfbf2af76d980/src/routes/responses/utils.ts#L176-L201 - */ -const itemHasImage = (item: OpenAIResponsesInputItem): boolean => { - const content = item.type === 'message' - ? item.content - : item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.output : undefined; - return Array.isArray(content) && content.some(part => part.type === 'input_image'); -}; - -export const withVisionHeaderSet = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - if (ctx.payload.input.some(itemHasImage)) ctx.headers.set('copilot-vision-request', 'true'); - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts b/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts deleted file mode 100644 index 271d5b2577..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import { collectOpenAIResponsesTools, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesTool, type OpenAIResponsesToolChoice } from '@floway-dev/protocols/openai-responses'; - -/** - * A Copilot gateway filters public `image_generation` from Responses requests, - * while OpenAI supports it. Apply this provider-specific rule after target - * selection to every declaration carrier and selector, retaining unrelated - * tools in their declaration containers and relative order. - * - * https://developers.openai.com/api/docs/guides/tools-image-generation - * https://github.com/caozhiyuan/copilot-api/blob/5d37d5b1ac6566c935a5c26d046396ee5fa423cc/src/routes/responses/handler.ts#L187-L204 - */ -const isImageGenerationTool = (tool: OpenAIResponsesTool): boolean => tool.type === 'image_generation'; - -const isImageGenerationToolChoice = (choice: OpenAIResponsesToolChoice | null | undefined): boolean => - typeof choice === 'object' && choice !== null && choice.type === 'image_generation'; - -const withoutToolChoice = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { - const { tool_choice: _choice, ...rest } = payload; - return rest; -}; - -export const stripImageGenerationFromPayload = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { - let removedTool = false; - const filter = (tools: OpenAIResponsesTool[]): OpenAIResponsesTool[] => { - const filtered = tools.filter(tool => { - const drop = isImageGenerationTool(tool); - removedTool ||= drop; - return !drop; - }); - return filtered.length === tools.length ? tools : filtered; - }; - let rewritten = payload; - if (Array.isArray(payload.tools)) { - const tools = filter(payload.tools); - if (tools.length === 0) { - const { tools: _tools, ...rest } = rewritten; - rewritten = rest; - } else if (tools !== payload.tools) rewritten = { ...rewritten, tools }; - } - const input = payload.input.map(item => { - if (item.type !== 'additional_tools' && item.type !== 'tool_search_output') return item; - const tools = filter(item.tools); - return tools === item.tools ? item : { ...item, tools }; - }); - if (input.some((item, index) => item !== payload.input[index])) rewritten = { ...rewritten, input }; - const choice = rewritten.tool_choice; - if (isImageGenerationToolChoice(choice)) { - return collectOpenAIResponsesTools(rewritten).length === 0 - ? withoutToolChoice(rewritten) : { ...rewritten, tool_choice: 'none' }; - } - if (typeof choice === 'object' && choice !== null && choice.type === 'allowed_tools') { - const allowed = choice.tools.filter(tool => tool.type !== 'image_generation'); - if (allowed.length === choice.tools.length) return rewritten; - return { ...rewritten, tool_choice: allowed.length === 0 && choice.mode === 'auto' ? 'none' : { ...choice, tools: allowed } }; - } - // Filtering a required choice must not expose tools outside its declared subset. - if (removedTool && choice === 'required' && collectOpenAIResponsesTools(rewritten).length === 0) return withoutToolChoice(rewritten); - return rewritten; -}; - -export const withImageGenerationStripped = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = stripImageGenerationFromPayload(ctx.payload); - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts b/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts deleted file mode 100644 index d454511c7f..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; - -/** - * Copilot has no request-side `service_tier`: `/responses` answers any value - * of the field with HTTP 400 - * `{"code":"unsupported_value","param":"service_tier"}`, on a base id and a - * `-fast` id alike. The tier reaches the upstream as the raw model id - * `callOpenAIResponses` selected instead, so by the time this runs the value - * has already been consumed and only the unusable field is left to drop. - * Strip it only after planning has committed to the OpenAI Responses target - * so source-side behavior and telemetry still see the caller's original - * request. Generic in the run-result type so the same definition feeds both - * the streaming `/responses` chain and the non-streaming compaction chain. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/commit/f7835a44f06976cab874700e4d94a5f5c0379369 - * - https://platform.openai.com/docs/api-reference/responses/create - */ -export const withServiceTierStripped = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - const { service_tier: _, ...payload } = ctx.payload; - ctx.payload = payload; - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/types.ts b/packages/provider-copilot/src/interceptors/openai-responses/types.ts deleted file mode 100644 index 96ddc9939a..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/types.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { Interceptor } from '@floway-dev/interceptor'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import type { ProviderModel, ProviderOpenAIResponsesResult, OpenAIResponsesAction } from '@floway-dev/provider'; - -// Boundary ctx for Copilot OpenAI Responses interceptors. See anthropic-messages/types.ts for -// the boundary-isolation rationale. A single chain wraps both the streaming -// `/responses` call and the non-streaming synth-via-trigger compaction call; -// the chain terminal dispatches on `ctx.action` to pick the wire shape. -// `action` mirrors the gateway-side OpenAIResponsesInvocation.action — interceptors -// MAY mutate it during the chain to re-route dispatch in the terminal. -export interface OpenAIResponsesBoundaryCtx { - payload: CanonicalOpenAIResponsesPayload; - headers: Headers; - readonly model: ProviderModel; - action: OpenAIResponsesAction; -} - -// Single chain feeds both the streaming generate terminal and the compact -// terminal; the terminal switches on `ctx.action` and emits the matching -// ProviderOpenAIResponsesResult variant. Pure payload/header mutators are written -// with a `` generic so they fit; event-stream mutators (whitespace -// abort, item-id membrane) inspect the result variant directly. -export type CopilotOpenAIResponsesBoundaryInterceptor = Interceptor< - OpenAIResponsesBoundaryCtx, - ProviderOpenAIResponsesResult ->; diff --git a/packages/provider-copilot/src/model-selection.ts b/packages/provider-copilot/src/model-selection.ts index 6fd9f19dc0..e170220c75 100644 --- a/packages/provider-copilot/src/model-selection.ts +++ b/packages/provider-copilot/src/model-selection.ts @@ -45,7 +45,7 @@ const firstPreferred = (models: readonly CopilotRawModel[]): CopilotRawModel | u // A narrowing filter that rolls back to the original pool when it would empty // it. On the OpenAI Responses path that rollback is the feature: OpenAI // answers an unavailable Fast mode by serving the standard lane and saying -// so, and `callOpenAIResponses` mirrors it. `callAnthropicMessages` never +// so, and OpenAI Responses selection mirrors it. Anthropic Messages selection never // reaches the rollback because Anthropic makes Fast Mode a hard contract and // the entry point pre-checks it. const narrow = (pool: readonly CopilotRawModel[], predicate: (model: CopilotRawModel) => boolean): readonly CopilotRawModel[] => { @@ -98,7 +98,7 @@ export const resolveCopilotRawModel = (models: CopilotModelsResponse, modelId: s }; // Whether the family can serve the accelerated lane at all. -// `callAnthropicMessages` pre-checks this because Anthropic rejects +// Anthropic Messages model selection pre-checks this because Anthropic rejects // `speed: 'fast'` on a model that cannot serve it, and Copilot never echoes // `usage.speed` for us to notice a downgrade afterwards. The OpenAI spelling // of the same lane needs no pre-check: that upstream reports the tier it diff --git a/packages/provider-copilot/src/interceptors/openai-responses/item-id-carrier.ts b/packages/provider-copilot/src/opaque-item-id.ts similarity index 100% rename from packages/provider-copilot/src/interceptors/openai-responses/item-id-carrier.ts rename to packages/provider-copilot/src/opaque-item-id.ts diff --git a/packages/provider-copilot/src/provider.ts b/packages/provider-copilot/src/provider.ts index e5a481b3ee..f4a9ab0f7b 100644 --- a/packages/provider-copilot/src/provider.ts +++ b/packages/provider-copilot/src/provider.ts @@ -1,30 +1,17 @@ import { chatFromCopilotRaw } from './chat-from-raw.ts'; -import { COMPACTION_TRIGGER, compactionResponse } from './compaction.ts'; import { assertCopilotUpstreamRecord } from './config.ts'; import { COPILOT_DEFAULT_FLAGS, defaultFlagsForCopilotModel } from './defaults.ts'; import { fetchCopilotModels } from './fetch-models.ts'; -import { copilotFetchOpenAIChatCompletions, copilotFetchOpenAIEmbeddings, copilotFetchAnthropicMessages, copilotFetchAnthropicMessagesCountTokens, copilotFetchOpenAIResponses, type CopilotDataPlaneFetchOptions } from './fetch.ts'; -import { COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY } from './interceptors/anthropic-messages/index.ts'; -import type { AnthropicMessagesBoundaryCtx } from './interceptors/anthropic-messages/types.ts'; -import { COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY } from './interceptors/openai-chat-completions/index.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from './interceptors/openai-chat-completions/types.ts'; -import { COPILOT_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; -import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { emptyKnownModels, mergeKnownModels, projectKnownModels } from './known-models.ts'; import { mergeCopilotVariants } from './merge-variants.ts'; -import { CONTEXT_1M_BETA, copilotModelSupportsFastVariant } from './model-selection.ts'; import { copilotVariantIndex } from './model-variants.ts'; -import { copilotOpenAIEmbeddingsBody, rawModelFor, rawModelSupportsEndpoint } from './operation-model.ts'; +import { rawModelSupportsEndpoint } from './operation-model.ts'; import { createCopilotPipelines } from './pipelines.ts'; import { pricingForCopilotPublicModelId } from './pricing.ts'; import { readCopilotUpstreamState, type CopilotUpstreamState } from './state.ts'; import type { CopilotRawModel } from './types.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import { parseAnthropicMessagesStream, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { type ModelEndpoints, type ProtocolFrame, isFastServiceTier, kindForEndpoints } from '@floway-dev/protocols/common'; -import { parseOpenAIChatCompletionsStream, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { parseOpenAIResponsesStream, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; -import { eventResult, getProviderRepo, headersForAnthropicMessagesCall, jsonRequestBody, readUpstreamApiError, streamingProviderCall, apiErrorToResponse, resolveEffectiveFlags, type ExecuteResult, type FetchInit, type FlagOverrides, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type TelemetryModelIdentity, type UpstreamCallOptions, type UpstreamRecord } from '@floway-dev/provider'; +import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; +import { getProviderRepo, resolveEffectiveFlags, type FlagOverrides, type ProviderInstance, type Provider, type ProviderModel, type UpstreamRecord } from '@floway-dev/provider'; interface CopilotProviderData { rawModels: CopilotRawModel[]; @@ -71,29 +58,6 @@ const copilotModelEndpoints = (rawModels: readonly CopilotRawModel[]): ModelEndp return rawModels.some(model => rawModelSupportsEndpoint(model, 'openaiEmbeddings')) ? { openaiEmbeddings: {} } : {}; }; -const chatReasoningEffort = (body: Omit): string | undefined => (body.reasoning_effort && body.reasoning_effort !== 'none' ? body.reasoning_effort : undefined); - -const anthropicMessagesReasoningEffort = (body: Omit): string | undefined => body.output_config?.effort; - -const openaiResponsesReasoningEffort = (body: Omit): string | undefined => (body.reasoning?.effort && body.reasoning.effort !== 'none' ? body.reasoning.effort : undefined); - -const anthropicMessagesBoundaryContext = ( - body: Omit, - model: ProviderModel, - headers: Headers, - anthropicBeta: readonly string[], -): AnthropicMessagesBoundaryCtx => { - return { - payload: { ...body, model: model.id }, - headers: new Headers(headers), - anthropicBeta: [...anthropicBeta], - model, - }; -}; - -const rejectUnsupported = (capability: string) => (): Promise => - Promise.reject(new Error(`Copilot provider does not implement ${capability}`)); - const finalizeCopilotModels = ( rawModels: CopilotRawModel[], upstreamOverrides: FlagOverrides, @@ -144,103 +108,7 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { const copilot = assertCopilotUpstreamRecord(record); const upstreamConfig = { id: copilot.id, githubHost: copilot.config.githubHost, githubToken: copilot.config.githubToken }; - const call = async ( - transport: (config: typeof upstreamConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions) => Promise, - body: Record, - signal: AbortSignal | undefined, - rawModel: CopilotRawModel, - headers: HttpHeaderLines, - opts: UpstreamCallOptions, - ): Promise => { - const response = await transport( - upstreamConfig, - { - method: 'POST', - body: jsonRequestBody({ ...body, model: rawModel.id }), - signal, - }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall, waitUntil: opts.waitUntil }, - ); - return { response, modelKey: rawModel.id }; - }; - - const callStreaming = ( - transport: (config: typeof upstreamConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions) => Promise, - body: Record, - signal: AbortSignal | undefined, - rawModel: CopilotRawModel, - headers: HttpHeaderLines, - parser: Parameters>[1], - opts: UpstreamCallOptions, - ) => - streamingProviderCall( - transport( - upstreamConfig, - { - method: 'POST', - body: jsonRequestBody({ ...body, stream: true, model: rawModel.id }), - signal, - }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall, waitUntil: opts.waitUntil }, - ), - parser, - rawModel.id, - signal, - ); - - // The boundary chain expects ExecuteResult shape so post-`run()` inspectors - // (e.g. rewriteContextWindowError) can pattern-match on `result.type`. The - // placeholder here only has to satisfy the EventResult contract while the - // chain runs inside the provider boundary; real telemetry identity is - // rebuilt downstream with pricing. - const placeholderIdentity = (modelKey: string): TelemetryModelIdentity => ({ - model: modelKey, - upstream: copilot.id, - modelKey, - pricing: null, - }); - - // Materialize an upstream error body up-front so any interceptor that - // inspects `result.body` (e.g. rewriteContextWindowError) sees the bytes. - const liftStream = async ( - streamPromise: Promise>, - ): Promise>> => { - const stream = await streamPromise; - if (stream.ok) { - return eventResult( - stream.events as AsyncIterable>, - placeholderIdentity(stream.modelKey), - { headers: stream.headers }, - ); - } - return await readUpstreamApiError(stream.response); - }; - - // Lowering rebuilds a ProviderStreamResult so callers continue to relay - // status/headers/body verbatim on errors and forward the typed event stream - // on success. `internal-error` is not a shape any Copilot boundary - // interceptor produces today; an explicit throw makes a future regression - // noisy instead of silently dropping the result. - const lowerToStream = ( - result: ExecuteResult>, - modelKey: string, - ): ProviderStreamResult => { - if (result.type === 'events') { - return { - ok: true, - events: result.events as AsyncIterable>, - modelKey, - ...(result.headers ? { headers: result.headers } : {}), - }; - } - if (result.type === 'api-error') { - return { ok: false, response: apiErrorToResponse(result), modelKey }; - } - throw new Error(`Copilot boundary chain produced unexpected ExecuteResult shape '${result.type}'`); - }; - const instance: ProviderInstance = { - callAlphaSearch: rejectUnsupported('callAlphaSearch'), getProvidedModels: async fetcher => { const fresh = await getProviderRepo().upstreams.getById(copilot.id); if (!fresh) throw new Error(`Copilot upstream ${copilot.id} disappeared mid-request`); @@ -267,166 +135,6 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { } return finalizeCopilotModels(projectKnownModels(merged, now), copilot.flagOverrides); }, - // Copilot's catalog never declares endpoints.openaiCompletions, so this - // stub is unreachable; the rejection surfaces a routing bug. - callOpenAICompletions: rejectUnsupported('callOpenAICompletions'), - callOpenAIChatCompletions: async (model, body, signal, opts) => { - // No accelerated-lane hint here on purpose. Copilot's /chat/completions - // accepts `service_tier` only to ignore it, answering - // `service_tier: "default"` even on a request that carried `priority`. - // Selecting a `-fast` raw variant from this entry point would have - // GitHub charge us the fast rates while the response reports the base - // tier, which is the one combination our billing cannot see through. - const rawModel = rawModelFor(model, 'openaiChatCompletions', { reasoningEffort: chatReasoningEffort(body) }); - const ctx: OpenAIChatCompletionsBoundaryCtx = { - payload: { ...body, model: model.id }, - headers: new Headers(opts.headers), - model, - }; - const result = await runInterceptors>>( - ctx, COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - return await liftStream(callStreaming(copilotFetchOpenAIChatCompletions, wireBody, signal, rawModel, [...ctx.headers], parseOpenAIChatCompletionsStream, opts)); - }, - ); - return lowerToStream(result, rawModel.id); - }, - callOpenAIResponses: async (model, body, action, signal, opts) => { - // `service_tier` naming the accelerated lane — `priority`, or its - // post-rename synonym `fast` — picks the family's `-fast` raw variant. - // Copilot rejects the field itself with HTTP 400 `unsupported_value` - // (see `strip-service-tier.ts`), so the raw id is the only way to reach - // the lane. Unlike Anthropic Messages below there is no hard pre-check: - // OpenAI declines an unavailable Fast mode silently and reports the tier - // it actually served, and Copilot does the same, so a family without a - // `-fast` variant falls back to its base and answers - // `service_tier: "default"`. The caller is told which tier ran and - // `billableUsageFromOpenAIResponsesResult` prices that one. - // https://github.com/openai/codex/issues/32191 - // https://learn.microsoft.com/en-sg/answers/questions/5921564/we-send-service-tier-priority-on-a-gpt-4-1-mini-gl - const rawModel = rawModelFor(model, 'openaiResponses', { - reasoningEffort: openaiResponsesReasoningEffort(body), - fast: isFastServiceTier(body.service_tier), - }); - const ctx: OpenAIResponsesBoundaryCtx = { - payload: { ...body, model: model.id }, - headers: new Headers(opts.headers), - model, - action, - }; - // Single chain wraps both branches; the terminal dispatches on - // `ctx.action` (the post-chain value), so a mid-chain interceptor can - // flip it and steer dispatch end-to-end. Copilot has no native - // /v1/responses/compact, so the compact branch drives the same - // /responses upstream with stream:false + a compaction_trigger input - // item and reshapes the envelope via `compactionResponse`. Every - // payload/header workaround in the chain — force-store-false, - // strip-service-tier, strip-image-generation, inline-image - // compression, vision/initiator headers — applies to both branches - // identically. The item-id membrane also normalizes the compact value - // envelope, while the whitespace guard only inspects generate streams. - return await runInterceptors( - ctx, COPILOT_OPENAI_RESPONSES_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - switch (ctx.action) { - case 'generate': { - const stream = await callStreaming(copilotFetchOpenAIResponses, wireBody, signal, rawModel, [...ctx.headers], parseOpenAIResponsesStream, opts); - return stream.ok - ? { action: 'generate', ok: true, events: stream.events, modelKey: stream.modelKey, ...(stream.headers ? { headers: stream.headers } : {}) } - : { action: 'generate', ok: false, response: stream.response, modelKey: stream.modelKey }; - } - case 'compact': { - const input = wireBody.input; - const triggered = { ...wireBody, input: [...input, COMPACTION_TRIGGER], stream: false, model: rawModel.id }; - const response = await copilotFetchOpenAIResponses( - upstreamConfig, - { method: 'POST', body: jsonRequestBody(triggered), signal }, - { extraHeaders: [...ctx.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall, waitUntil: opts.waitUntil }, - ); - if (!response.ok) return { action: 'compact', ok: false, response, modelKey: rawModel.id }; - const generated = (await response.json()) as OpenAIResponsesResult; - return { action: 'compact', ok: true, result: compactionResponse(input, generated), modelKey: rawModel.id }; - } - default: - ctx.action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${ctx.action as string}`); - } - }, - ); - }, - callAnthropicMessages: async (model, body, signal, opts) => { - // Fast Mode is a hard contract on the request side: Anthropic returns - // HTTP 400 invalid_request_error when a model does not support it, with - // no silent fallback to standard speed. We mirror that at the gateway - // boundary before any per-Copilot workaround runs — selection alone is - // best-effort, and Copilot never echoes `usage.speed`, so an unchecked - // downgrade would be invisible to the caller and to billing alike. The - // OpenAI spelling of the same lane takes the opposite path in - // `callOpenAIResponses`, where the upstream reports the tier it served. - // https://docs.claude.com/en/build-with-claude/fast-mode - // - // The `error.message` is byte-identical to the string Anthropic emits - // on the real wire, recorded verbatim from a live response by an - // independent gateway's regression test: - // https://github.com/Yeachan-Heo/gajae-code/blob/main/packages/ai/test/anthropic-fast-mode.test.ts - if (body.speed === 'fast') { - const providerData = model.providerData as CopilotProviderData; - if (!copilotModelSupportsFastVariant(providerData.rawModels)) { - return { - ok: false, - response: Response.json( - { - type: 'error', - error: { - type: 'invalid_request_error', - message: `'${model.id}' does not support the \`speed\` parameter.`, - }, - }, - { status: 400 }, - ), - modelKey: model.id, - }; - } - } - - // Both the native Anthropic Messages call and count_tokens select the same raw - // `messages` variant; they differ only in the upstream endpoint path. - const ctx = anthropicMessagesBoundaryContext(body, model, opts.headers, opts.anthropicBeta); - const rawModel = rawModelFor(model, 'anthropicMessages', { - context1m: ctx.anthropicBeta.includes(CONTEXT_1M_BETA), - reasoningEffort: anthropicMessagesReasoningEffort(body), - fast: body.speed === 'fast', - }); - const result = await runInterceptors>>( - ctx, COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - return await liftStream(callStreaming(copilotFetchAnthropicMessages, wireBody, signal, rawModel, headersForAnthropicMessagesCall([...ctx.headers], ctx.anthropicBeta), parseAnthropicMessagesStream, opts)); - }, - ); - return lowerToStream(result, rawModel.id); - }, - callAnthropicMessagesCountTokens: async (model, body, signal, opts) => { - const ctx = anthropicMessagesBoundaryContext(body, model, opts.headers, opts.anthropicBeta); - const rawModel = rawModelFor(model, 'anthropicMessages', { - context1m: ctx.anthropicBeta.includes(CONTEXT_1M_BETA), - reasoningEffort: anthropicMessagesReasoningEffort(body), - }); - const response = await runInterceptors( - ctx, COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - const { response } = await call(copilotFetchAnthropicMessagesCountTokens, wireBody, signal, rawModel, headersForAnthropicMessagesCall([...ctx.headers], ctx.anthropicBeta), opts); - return response; - }, - ); - return { response, modelKey: rawModel.id }; - }, - callOpenAIEmbeddings: (model, body, signal, opts) => call(copilotFetchOpenAIEmbeddings, copilotOpenAIEmbeddingsBody(body), signal, rawModelFor(model, 'openaiEmbeddings'), [...opts.headers], opts), - // Copilot has no /images/* upstream; catalog never emits a kind='image' - // model, so these stubs are unreachable. - callOpenAIImagesGenerations: rejectUnsupported('callOpenAIImagesGenerations'), - callOpenAIImagesEdits: rejectUnsupported('callOpenAIImagesEdits'), - callOpenAIAudioTranscriptions: rejectUnsupported('callOpenAIAudioTranscriptions'), - callRerank: rejectUnsupported('callRerank'), }; return { diff --git a/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts b/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts index 9e48309b0a..fd0ae5af88 100644 --- a/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts @@ -1,5 +1,5 @@ import type { MessagesFacts } from '../../chat-facts.ts'; -import { memoizedBase64Compressor } from '../../interceptors/image-compression.ts'; +import { memoizedBase64Compressor } from '../../image-compression.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { type ImageSizeCalculator, type SizeCaps, fitWithin } from '@floway-dev/platform'; import type { AnthropicMessagesPayload, AnthropicMessagesImageBlock, AnthropicMessagesMessage, AnthropicMessagesToolResultBlock, AnthropicMessagesToolResultContentBlock, AnthropicMessagesUserContentBlock } from '@floway-dev/protocols/anthropic-messages'; diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts index 2b7edb218d..aafe51ccd5 100644 --- a/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts @@ -1,6 +1,6 @@ import { CLAUDE_AGENT_USER_AGENT } from '../../auth.ts'; import type { MessagesFacts } from '../../chat-facts.ts'; -import { parseUserIdMetadata } from '../../interceptors/anthropic-messages/detect-claude-code-metadata.ts'; +import { parseUserIdMetadata } from '../../claude-code-metadata.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { replaceHttpHeader } from '@floway-dev/provider'; diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts index 777191a80b..841a7916fe 100644 --- a/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts @@ -1,7 +1,7 @@ import { v4 } from 'uuid'; import type { MessagesFacts } from '../../chat-facts.ts'; -import { parseUserIdMetadata } from '../../interceptors/anthropic-messages/detect-claude-code-metadata.ts'; +import { parseUserIdMetadata } from '../../claude-code-metadata.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { replaceHttpHeader } from '@floway-dev/provider'; diff --git a/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts index c98c5b88ce..96aa785364 100644 --- a/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts +++ b/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts @@ -1,5 +1,5 @@ -import { checkWhitespaceOverflow } from '../../interceptors/shared/whitespace-overflow.ts'; +import { checkWhitespaceOverflow } from '../../tool-argument-whitespace.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; diff --git a/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts b/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts index cdd83cfe20..43594b1665 100644 --- a/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts +++ b/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts @@ -1,6 +1,6 @@ import type { ChatCompletionsFacts } from '../../chat-facts.ts'; -import { memoizedDataUrlCompressor } from '../../interceptors/image-compression.ts'; -import { targetSizeForOpenAIResponsesChat } from '../../interceptors/image-size.ts'; +import { memoizedDataUrlCompressor } from '../../image-compression.ts'; +import { targetSizeForOpenAIResponsesChat } from '../../image-size.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsContentPart, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; import { type ProviderModelFacts } from '@floway-dev/provider'; diff --git a/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts index 527b9a14ef..fc0c3b8332 100644 --- a/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts +++ b/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts @@ -1,5 +1,5 @@ -import { checkWhitespaceOverflow } from '../../interceptors/shared/whitespace-overflow.ts'; +import { checkWhitespaceOverflow } from '../../tool-argument-whitespace.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; diff --git a/packages/provider-copilot/src/stages/openai-responses/compress-images.ts b/packages/provider-copilot/src/stages/openai-responses/compress-images.ts index b8e4c0bd4e..c0068cec80 100644 --- a/packages/provider-copilot/src/stages/openai-responses/compress-images.ts +++ b/packages/provider-copilot/src/stages/openai-responses/compress-images.ts @@ -1,6 +1,6 @@ import type { ResponsesFacts } from '../../chat-facts.ts'; -import { memoizedDataUrlCompressor } from '../../interceptors/image-compression.ts'; -import { targetSizeForOpenAIResponsesChat } from '../../interceptors/image-size.ts'; +import { memoizedDataUrlCompressor } from '../../image-compression.ts'; +import { targetSizeForOpenAIResponsesChat } from '../../image-size.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputImage, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; import { type ProviderModelFacts } from '@floway-dev/provider'; diff --git a/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts b/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts index 2b32136705..329482a856 100644 --- a/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts +++ b/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts @@ -1,5 +1,5 @@ import type { ResponsesFacts } from '../../chat-facts.ts'; -import { unwrapCopilotItemId, wrapCopilotItemId } from '../../interceptors/openai-responses/item-id-carrier.ts'; +import { unwrapCopilotItemId, wrapCopilotItemId } from '../../opaque-item-id.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { encodeHex, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; diff --git a/packages/provider-copilot/src/interceptors/shared/whitespace-overflow.ts b/packages/provider-copilot/src/tool-argument-whitespace.ts similarity index 92% rename from packages/provider-copilot/src/interceptors/shared/whitespace-overflow.ts rename to packages/provider-copilot/src/tool-argument-whitespace.ts index f34fe08365..c91462cfde 100644 --- a/packages/provider-copilot/src/interceptors/shared/whitespace-overflow.ts +++ b/packages/provider-copilot/src/tool-argument-whitespace.ts @@ -3,7 +3,7 @@ * line breaks / tabs in function `arguments` until `max_tokens`, which keeps * the client hanging while never producing valid JSON. * - * Two interceptors (OpenAI Chat Completions and OpenAI Responses) share this counter and + * Two stages (OpenAI Chat Completions and OpenAI Responses) share this counter and * threshold so the defense fires the same way on both upstream shapes. * * References: diff --git a/packages/provider-custom/__tests__/stages/audio-request_test.ts b/packages/provider-custom/__tests__/stages/audio-request_test.ts new file mode 100644 index 0000000000..1631b6f68d --- /dev/null +++ b/packages/provider-custom/__tests__/stages/audio-request_test.ts @@ -0,0 +1,23 @@ +import { expect, test } from 'vitest'; + +import { prepareCustomRequest } from '../../src/stages/prepare-request.ts'; +import { http } from '@floway-dev/http/pipeline'; +import type { HttpMultipartBody } from '@floway-dev/http/request-content'; +import { compose } from '@floway-dev/pipeline'; +import { observeProviderCall, providerModelFacts, type ProviderRequest, type ProviderResponse } from '@floway-dev/provider'; +import { collectHttpPipeline, noopUpstreamCallOptions, stubProviderModel } from '@floway-dev/test-utils'; + +test('audio prepare preserves ordered fields and file metadata while replacing the selected model', async () => { + const file = { bytes: new Uint8Array([1, 2, 3, 4]), name: 'meeting.wav', type: 'audio/wav', lastModified: 1_700_000_000_000 }; + const entries = [{ name: 'file', value: file }, { name: 'language', value: 'en' }, { name: 'model', value: 'public-model' }, { name: 'timestamp_granularities[]', value: 'word' }, { name: 'timestamp_granularities[]', value: 'segment' }]; + const prepare = prepareCustomRequest({ baseUrl: 'https://custom.example', authStyle: 'none', endpoints: {}, ingressHeadersRules: [], modelsFetch: { enabled: false }, models: [] }, 'openaiAudioTranscriptions'); + const pipeline = compose, ProviderResponse & Record>('audio.metadata', [prepare, observeProviderCall, http]); + const executed = await collectHttpPipeline(pipeline, { 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: 'upstream-model' })), 'request.provider.payload': { entries }, 'request.http.callId': 0, 'request.http.headers': [] }, { httpCall: () => ({ ...noopUpstreamCallOptions(), fetcher: async () => new Response('{}'), signal: undefined }) }); + const body = executed.facts['request.http.body'] as HttpMultipartBody; + expect(body.entries.map(entry => entry.name)).toEqual(['file', 'language', 'model', 'timestamp_granularities[]', 'timestamp_granularities[]']); + expect(body.entries[0]!.value).toBe(file); + expect(body.entries[0]!.value).toMatchObject({ name: 'meeting.wav', type: 'audio/wav', lastModified: file.lastModified, bytes: new Uint8Array([1, 2, 3, 4]) }); + expect(body.entries[2]!.value).toBe('upstream-model'); + expect(body.entries.slice(3).map(entry => entry.value)).toEqual(['word', 'segment']); + expect(entries[2]!.value).toBe('public-model'); +}); diff --git a/packages/provider-custom/__tests__/fetch_test.ts b/packages/provider-custom/__tests__/stages/prepare-request_test.ts similarity index 69% rename from packages/provider-custom/__tests__/fetch_test.ts rename to packages/provider-custom/__tests__/stages/prepare-request_test.ts index 62a9b52bc8..38e89967c0 100644 --- a/packages/provider-custom/__tests__/fetch_test.ts +++ b/packages/provider-custom/__tests__/stages/prepare-request_test.ts @@ -1,21 +1,12 @@ import { test } from 'vitest'; -import { assertCustomUpstreamRecord } from '../src/config.ts'; -import { - customFetchAlphaSearch, - customFetchOpenAIAudioTranscriptions, - customFetchOpenAIChatCompletions, - customFetchOpenAIEmbeddings, - customFetchAnthropicMessages, - customFetchAnthropicMessagesCountTokens, - customFetchModels, - customFetchOpenAIResponses, - customFetchOpenAIResponsesCompact, -} from '../src/fetch.ts'; -import { createCustomProvider } from '../src/provider.ts'; +import { assertCustomUpstreamRecord } from '../../src/config.ts'; +import { customFetchModels } from '../../src/control-http.ts'; +import { createCustomProvider } from '../../src/provider.ts'; +import { prepareCustomRequest } from '../../src/stages/prepare-request.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; -import { directFetcher, identityWrapUpstreamCall } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertExists, jsonResponse, noopUpstreamCallOptions, withMockedFetch } from '@floway-dev/test-utils'; +import { identityWrapUpstreamCall } from '@floway-dev/provider'; +import { collectPreparedProviderRequest, testFetcher, callProviderPipeline, assertEquals, assertExists, jsonResponse, noopUpstreamCallOptions, withMockedFetch } from '@floway-dev/test-utils'; const baseRecord: UpstreamRecord = { id: 'up_test', @@ -51,15 +42,15 @@ test('typed transports use default /v1/* paths', async () => { return new Response('{}', { status: 200 }); }, async () => { - await customFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIResponsesCompact(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchAnthropicMessagesCountTokens(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchAlphaSearch(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIEmbeddings(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIAudioTranscriptions(config, { method: 'POST', body: new FormData() }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchModels(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiResponses'), 'openaiResponses', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiResponsesCompact'), 'openaiResponsesCompact', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessagesCountTokens'), 'anthropicMessagesCountTokens', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'alphaSearch'), 'alphaSearch', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiEmbeddings'), 'openaiEmbeddings', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiAudioTranscriptions'), 'openaiAudioTranscriptions', key => key, { method: 'POST', body: new FormData() }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await customFetchModels(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -95,13 +86,13 @@ test('admin pathOverrides replace defaults and propagate to derived sub-paths', return new Response('{}', { status: 200 }); }, async () => { - await customFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); // count_tokens / compact follow their parent override. - await customFetchAnthropicMessagesCountTokens(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIResponsesCompact(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchAlphaSearch(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessagesCountTokens'), 'anthropicMessagesCountTokens', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiResponsesCompact'), 'openaiResponsesCompact', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'alphaSearch'), 'alphaSearch', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); // Endpoints without an override fall back to the OpenAI default. - await customFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -129,7 +120,7 @@ test('customFetchModels resolves the path from modelsFetch.endpoint', async () = return new Response('{}', { status: 200 }); }, async () => { - await customFetchModels(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await customFetchModels(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -151,7 +142,7 @@ test('customFetchModels falls back to the default /v1/models path when modelsFet return new Response('{}', { status: 200 }); }, async () => { - await customFetchModels(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await customFetchModels(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -169,7 +160,7 @@ test('bearer authStyle sends the configured token via Authorization', async () = return new Response('{}', { status: 200 }); }, async () => { - await customFetchModels(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await customFetchModels(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -196,7 +187,7 @@ test('authStyle "anthropic" sends x-api-key + anthropic-version', async () => { return new Response('{}', { status: 200 }); }, async () => { - await customFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -220,11 +211,7 @@ test('authStyle "anthropic" preserves a caller-supplied anthropic-version', asyn return new Response('{}', { status: 200 }); }, async () => { - await customFetchAnthropicMessages( - config, - { method: 'POST', body: '{}', headers: { 'anthropic-version': '2024-01-01' } }, - { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }, - ); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}', headers: { 'anthropic-version': '2024-01-01' } }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -252,7 +239,7 @@ test('authStyle "none" sends neither Authorization nor x-api-key', async () => { return new Response('{}', { status: 200 }); }, async () => { - await customFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -285,7 +272,7 @@ test('Custom provider callOpenAIImagesEdits forwards multipart body with model f }, async () => { const provider = createCustomProvider(record); - const [model] = await provider.instance.getProvidedModels(directFetcher); + const [model] = await provider.instance.getProvidedModels(testFetcher); const result = await callProviderPipeline(provider, 'openaiImagesEdits', model, { parameters: { prompt: 'add a kite' }, images: [{ @@ -325,7 +312,7 @@ test('Custom provider callOpenAIAudioTranscriptions preserves multipart entries }, async () => { const provider = createCustomProvider(record); - const [model] = await provider.instance.getProvidedModels(directFetcher); + const [model] = await provider.instance.getProvidedModels(testFetcher); const result = await callProviderPipeline(provider, 'openaiAudioTranscriptions', model, { entries: [ { name: 'file', value: { bytes: new Uint8Array([7, 8]), name: 'voice.ogg', type: 'audio/ogg' } }, diff --git a/packages/provider-custom/src/control-http.ts b/packages/provider-custom/src/control-http.ts new file mode 100644 index 0000000000..daaf99554c --- /dev/null +++ b/packages/provider-custom/src/control-http.ts @@ -0,0 +1,53 @@ +import type { CustomUpstreamConfig } from './config.ts'; +import { type FetchInit, type UpstreamFetchOptions, joinBaseAndPath } from '@floway-dev/provider'; + +// https://docs.anthropic.com/en/api/versioning +const ANTHROPIC_VERSION = '2023-06-01'; + +// The header block is carried as field lines rather than a `Headers`, because +// a rule set may send one name several times and `Headers.append` merges a +// repeated name into one value on undici. +// https://github.com/nodejs/undici/blob/v8.3.0/lib/web/fetch/headers.js#L236-L258 +const has = (lines: readonly (readonly [string, string])[], name: string): boolean => + lines.some(([candidate]) => candidate.toLowerCase() === name); + +const replace = (lines: [string, string][], name: string, value: string): void => { + const kept = lines.filter(([candidate]) => candidate.toLowerCase() !== name.toLowerCase()); + lines.length = 0; + lines.push(...kept, [name, value]); +}; + +const customFetchInternal = async ( + config: CustomUpstreamConfig, + path: string, + init: FetchInit, + options: UpstreamFetchOptions, +): Promise => { + const headers: [string, string][] = init.headers ? [...new Headers(init.headers)] : []; + if (config.authStyle === 'anthropic') { + replace(headers, 'x-api-key', config.apiKey); + if (!has(headers, 'anthropic-version')) headers.push(['anthropic-version', ANTHROPIC_VERSION]); + } else if (config.authStyle === 'bearer') { + replace(headers, 'Authorization', `Bearer ${config.apiKey}`); + } + // authStyle === 'none' falls through with no auth header. The same goes + // for the /models fetch — Models Fetch shares this code path, so a 'none' + // upstream is queried anonymously end-to-end. + if (init.body && !has(headers, 'content-type') && !(init.body instanceof FormData)) { + headers.push(['Content-Type', 'application/json']); + } + if (options.extraHeaders) { + // Every line the caller supplies for a name replaces this helper's own, + // and the caller's lines keep their order and their repetitions. + const supplied = new Set(options.extraHeaders.map(([name]) => name.toLowerCase())); + const kept = headers.filter(([name]) => !supplied.has(name.toLowerCase())); + headers.length = 0; + headers.push(...kept, ...options.extraHeaders.map(([name, value]): [string, string] => [name, value])); + } + return await options.wrapUpstreamCall(() => options.fetcher(joinBaseAndPath(config.baseUrl, path), { ...init, headers })); +}; + +// /models lives on its own fetch toggle (see config.modelsFetch.endpoint), +// not in pathOverrides. +export const customFetchModels = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => + customFetchInternal(config, config.modelsFetch.endpoint ?? '/v1/models', init, options); diff --git a/packages/provider-custom/src/fetch-models.ts b/packages/provider-custom/src/fetch-models.ts index 2e490354d6..f383c7c96f 100644 --- a/packages/provider-custom/src/fetch-models.ts +++ b/packages/provider-custom/src/fetch-models.ts @@ -10,7 +10,7 @@ // effort metadata. The container is admitted if `data` is an array. import type { CustomUpstreamConfig } from './config.ts'; -import { customFetchModels } from './fetch.ts'; +import { customFetchModels } from './control-http.ts'; import { BILLING_METRICS, canonicalizePricingSelector, type BillingMetric, type ModelKind, type ModelPricing, type OpaqueBlobCompatibilityScope, parseNonNegativeDecimalString, type PriceVector, type PricingSelector, validateModelPricing } from '@floway-dev/protocols/common'; import { chatField, fetchUpstreamModels, type Fetcher, type UpstreamChatModelConfig, identityWrapUpstreamCall, opaqueBlobCompatibilityScopeField } from '@floway-dev/provider'; diff --git a/packages/provider-custom/src/fetch.ts b/packages/provider-custom/src/fetch.ts deleted file mode 100644 index 5df603f6f9..0000000000 --- a/packages/provider-custom/src/fetch.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { CustomPathOverrideKey, CustomUpstreamConfig } from './config.ts'; -import { type FetchInit, type UpstreamFetchOptions, joinBaseAndPath } from '@floway-dev/provider'; - -// https://docs.anthropic.com/en/api/versioning -const ANTHROPIC_VERSION = '2023-06-01'; - -// Endpoint key is the OpenAI-canonical path fragment (`/chat/completions`, -// `/images/generations`, ...). The default upstream URL is the key prefixed -// with `/v1`; pathOverrides (see config.ts) replace it one-for-one. The -// messages count-tokens and responses compact endpoints append a suffix -// to their parent's resolved path so an override of the parent ripples -// down to both. -const pathOverrideFor = (config: CustomUpstreamConfig, key: CustomPathOverrideKey): string => - config.pathOverrides?.[key] ?? `/v1${key}`; - -// The header block is carried as field lines rather than a `Headers`, because -// a rule set may send one name several times and `Headers.append` merges a -// repeated name into one value on undici. -// https://github.com/nodejs/undici/blob/v8.3.0/lib/web/fetch/headers.js#L236-L258 -const has = (lines: readonly (readonly [string, string])[], name: string): boolean => - lines.some(([candidate]) => candidate.toLowerCase() === name); - -const replace = (lines: [string, string][], name: string, value: string): void => { - const kept = lines.filter(([candidate]) => candidate.toLowerCase() !== name.toLowerCase()); - lines.length = 0; - lines.push(...kept, [name, value]); -}; - -const customFetchInternal = async ( - config: CustomUpstreamConfig, - path: string, - init: FetchInit, - options: UpstreamFetchOptions, -): Promise => { - const headers: [string, string][] = init.headers ? [...new Headers(init.headers)] : []; - if (config.authStyle === 'anthropic') { - replace(headers, 'x-api-key', config.apiKey); - if (!has(headers, 'anthropic-version')) headers.push(['anthropic-version', ANTHROPIC_VERSION]); - } else if (config.authStyle === 'bearer') { - replace(headers, 'Authorization', `Bearer ${config.apiKey}`); - } - // authStyle === 'none' falls through with no auth header. The same goes - // for the /models fetch — Models Fetch shares this code path, so a 'none' - // upstream is queried anonymously end-to-end. - if (init.body && !has(headers, 'content-type') && !(init.body instanceof FormData)) { - headers.push(['Content-Type', 'application/json']); - } - if (options.extraHeaders) { - // Every line the caller supplies for a name replaces this helper's own, - // and the caller's lines keep their order and their repetitions. - const supplied = new Set(options.extraHeaders.map(([name]) => name.toLowerCase())); - const kept = headers.filter(([name]) => !supplied.has(name.toLowerCase())); - headers.length = 0; - headers.push(...kept, ...options.extraHeaders.map(([name, value]): [string, string] => [name, value])); - } - return await options.wrapUpstreamCall(() => options.fetcher(joinBaseAndPath(config.baseUrl, path), { ...init, headers })); -}; - -export const customFetchRerank = (config: CustomUpstreamConfig, path: string, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, path, init, options); - -export const customFetchOpenAIChatCompletions = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/chat/completions'), init, options); -export const customFetchOpenAIResponses = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/responses'), init, options); -export const customFetchOpenAIResponsesCompact = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, `${pathOverrideFor(config, '/responses')}/compact`, init, options); -export const customFetchAnthropicMessages = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/messages'), init, options); -export const customFetchAnthropicMessagesCountTokens = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, `${pathOverrideFor(config, '/messages')}/count_tokens`, init, options); -export const customFetchOpenAIEmbeddings = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/embeddings'), init, options); -export const customFetchOpenAICompletions = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/completions'), init, options); -export const customFetchOpenAIImagesGenerations = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/images/generations'), init, options); -export const customFetchOpenAIImagesEdits = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/images/edits'), init, options); -export const customFetchOpenAIAudioTranscriptions = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/audio/transcriptions'), init, options); -export const customFetchAlphaSearch = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/alpha/search'), init, options); -// /models lives on its own fetch toggle (see config.modelsFetch.endpoint), -// not in pathOverrides. -export const customFetchModels = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, config.modelsFetch.endpoint ?? '/v1/models', init, options); diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index 7742d6a6dd..de3b06ca0d 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -1,15 +1,10 @@ -import { assertCustomUpstreamRecord, type CustomUpstreamConfig } from './config.ts'; +import { assertCustomUpstreamRecord } from './config.ts'; import { CUSTOM_DEFAULT_FLAGS } from './defaults.ts'; import { fetchCustomModels, type CustomModelsResponse, type CustomRawModel } from './fetch-models.ts'; -import { customFetchAlphaSearch, customFetchOpenAIAudioTranscriptions, customFetchOpenAIChatCompletions, customFetchOpenAICompletions, customFetchOpenAIEmbeddings, customFetchOpenAIImagesEdits, customFetchOpenAIImagesGenerations, customFetchAnthropicMessages, customFetchAnthropicMessagesCountTokens, customFetchRerank, customFetchOpenAIResponses, customFetchOpenAIResponsesCompact } from './fetch.ts'; import { inferEndpointsFromModelId } from './infer-endpoints.ts'; import { createCustomPipelines } from './pipelines.ts'; -import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; -import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; -import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResult, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; -import { DEFAULT_RERANK_PATHS, serializeRerankRequest } from '@floway-dev/protocols/rerank'; -import { headersForAnthropicMessagesCall, jsonRequestBody, serializeModelFieldOpenAIAudioTranscriptionRequest, serializeOpenAIImagesEditsRequest, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FetchInit, type FlagId, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; +import { publicModelId, resolveEffectiveFlags, type FlagId, type ProviderInstance, type Provider, type ProviderModel, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; @@ -151,131 +146,12 @@ export const projectCustomModels = ( export const createCustomProvider = (record: UpstreamRecord): Provider => { const { config } = assertCustomUpstreamRecord(record); - // Each name is resolved as a whole: the admitted client values are dropped - // and its rules rebuild the name's value list in rule order, so a - // passthrough rule reinstates what the client sent and every configured - // rule contributes its own value beside it. - const valuesByKey = config.ingressHeadersRules.reduce>((byKey, rule) => { - const values = byKey.get(rule.key); - if (values) values.push(rule.value); - else byKey.set(rule.key, [rule.value]); - return byKey; - }, new Map()); - - const headersForCall = (headers: Headers): HttpHeaderLines => { - const resolved: [string, string][] = []; - for (const [name, value] of headers) { - if (!valuesByKey.has(name.toLowerCase())) resolved.push([name, value]); - } - for (const [key, values] of valuesByKey) { - const admitted = headers.get(key); - for (const value of values) { - if (value !== null) resolved.push([key, value]); - else if (admitted !== null) resolved.push([key, admitted]); - } - } - return resolved; - }; - const call = ( - transport: (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions) => Promise, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - opts: UpstreamCallOptions, - ): Promise => { - const rawModelId = rawModelIdOf(model); - return transport(config, { method: 'POST', body: jsonRequestBody({ ...body, model: rawModelId }), signal }, { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }) - .then(response => ({ - response, - modelKey: rawModelId, - })); - }; - - const callStreaming = ( - transport: (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions) => Promise, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - parser: ProviderStreamParser, - opts: UpstreamCallOptions, - ) => { - const rawModelId = rawModelIdOf(model); - return streamingProviderCall( - transport( - config, - { method: 'POST', body: jsonRequestBody({ ...body, stream: true, model: rawModelId }), signal }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ), - parser, - rawModelId, - signal, - ); - }; - const instance: ProviderInstance = { getProvidedModels: async fetcher => { if (!config.modelsFetch.enabled) return projectCustomModels(record); const response = await fetchCustomModels(config, fetcher); return projectCustomModels(record, response); }, - callAlphaSearch: (model, body, signal, opts) => call(customFetchAlphaSearch, model, body, signal, headersForCall(opts.headers), opts), - callOpenAICompletions: (model, body, signal, opts) => call(customFetchOpenAICompletions, model, body, signal, headersForCall(opts.headers), opts), - callOpenAIChatCompletions: (model, body, signal, opts) => callStreaming(customFetchOpenAIChatCompletions, model, body, signal, headersForCall(opts.headers), parseOpenAIChatCompletionsStream, opts), - callOpenAIResponses: async (model, body, action, signal, opts) => { - switch (action) { - case 'generate': { - const stream = await callStreaming(customFetchOpenAIResponses, model, body, signal, headersForCall(opts.headers), parseOpenAIResponsesStream, opts); - return stream.ok - ? { action: 'generate', ok: true, events: stream.events, modelKey: stream.modelKey, ...(stream.headers ? { headers: stream.headers } : {}) } - : { action: 'generate', ok: false, response: stream.response, modelKey: stream.modelKey }; - } - case 'compact': { - const rawModelId = rawModelIdOf(model); - const response = await customFetchOpenAIResponsesCompact( - config, - { method: 'POST', body: jsonRequestBody({ ...toCompactPayloadShape(body), model: rawModelId }), signal }, - { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return response.ok - ? { action: 'compact', ok: true, result: (await response.json()) as OpenAIResponsesCompactionResult, modelKey: rawModelId } - : { action: 'compact', ok: false, response, modelKey: rawModelId }; - } - default: - action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${action as string}`); - } - }, - callAnthropicMessages: (model, body, signal, opts) => callStreaming(customFetchAnthropicMessages, model, body, signal, headersForAnthropicMessagesCall(headersForCall(opts.headers), opts.anthropicBeta), parseAnthropicMessagesStream, opts), - callAnthropicMessagesCountTokens: (model, body, signal, opts) => call(customFetchAnthropicMessagesCountTokens, model, body, signal, headersForAnthropicMessagesCall(headersForCall(opts.headers), opts.anthropicBeta), opts), - callOpenAIEmbeddings: (model, body, signal, opts) => call(customFetchOpenAIEmbeddings, model, body, signal, headersForCall(opts.headers), opts), - callOpenAIImagesGenerations: (model, body, signal, opts) => call(customFetchOpenAIImagesGenerations, model, body, signal, headersForCall(opts.headers), opts), - callOpenAIImagesEdits: async (model, request, signal, opts) => { - const rawModelId = rawModelIdOf(model); - const body = await serializeOpenAIImagesEditsRequest(request, rawModelId); - const response = await customFetchOpenAIImagesEdits(config, { method: 'POST', body, signal }, { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: rawModelId }; - }, - callOpenAIAudioTranscriptions: async (model, request, signal, opts) => { - const rawModelId = rawModelIdOf(model); - const body = serializeModelFieldOpenAIAudioTranscriptionRequest(request, rawModelId); - const response = await customFetchOpenAIAudioTranscriptions(config, { method: 'POST', body, signal }, { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: rawModelId }; - }, - callRerank: async (model, request, signal, opts) => { - const target = model.rerankTarget; - if (target === undefined) throw new Error(`Rerank model ${model.id} has no outbound target`); - const rawModelId = rawModelIdOf(model); - const body = serializeRerankRequest(target.protocol, rawModelId, request); - const response = await customFetchRerank( - config, - target.path ?? DEFAULT_RERANK_PATHS[target.protocol], - { method: 'POST', body: jsonRequestBody(body), signal }, - { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return { response, modelKey: rawModelId, target }; - }, }; return { diff --git a/packages/provider-ollama/__tests__/provider_test.ts b/packages/provider-ollama/__tests__/provider_test.ts index d0d3d36a23..d660bed4a1 100644 --- a/packages/provider-ollama/__tests__/provider_test.ts +++ b/packages/provider-ollama/__tests__/provider_test.ts @@ -196,8 +196,6 @@ test('call* methods POST to /v1/ with the upstream model id and Bearer if (url.pathname === '/v1/chat/completions') { chatRequest = request; chatBody = await request.json(); - // SSE response so streamingProviderCall does not throw on the empty - // body — we only assert the request shape. return new Response('', { status: 200, headers: { 'content-type': 'text/event-stream' } }); } return new Response('unexpected', { status: 500 }); diff --git a/packages/provider-ollama/__tests__/fetch_test.ts b/packages/provider-ollama/__tests__/stages/prepare-request_test.ts similarity index 50% rename from packages/provider-ollama/__tests__/fetch_test.ts rename to packages/provider-ollama/__tests__/stages/prepare-request_test.ts index a7ccea17dd..7dda74f1a5 100644 --- a/packages/provider-ollama/__tests__/fetch_test.ts +++ b/packages/provider-ollama/__tests__/stages/prepare-request_test.ts @@ -1,19 +1,11 @@ import { test } from 'vitest'; -import { assertOllamaUpstreamRecord } from '../src/config.ts'; -import { - ollamaFetchOpenAIChatCompletions, - ollamaFetchOpenAIAudioTranscriptions, - ollamaFetchOpenAIEmbeddings, - ollamaFetchAnthropicMessages, - ollamaFetchOpenAIResponses, - ollamaFetchOpenAIResponsesCompact, - ollamaFetchShow, - ollamaFetchTags, -} from '../src/fetch.ts'; +import { assertOllamaUpstreamRecord } from '../../src/config.ts'; +import { ollamaFetchShow, ollamaFetchTags } from '../../src/control-http.ts'; +import { prepareOllamaRequest } from '../../src/stages/prepare-request.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; -import { directFetcher, identityWrapUpstreamCall } from '@floway-dev/provider'; -import { assertEquals, withMockedFetch } from '@floway-dev/test-utils'; +import { identityWrapUpstreamCall } from '@floway-dev/provider'; +import { collectPreparedProviderRequest, testFetcher, assertEquals, withMockedFetch } from '@floway-dev/test-utils'; const baseRecord: UpstreamRecord = { id: 'up_ollama_test', @@ -45,14 +37,14 @@ test('typed transports hit the fixed Ollama endpoint paths', async () => { return new Response('{}', { status: 200 }); }, async () => { - await ollamaFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchOpenAIResponsesCompact(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchOpenAIEmbeddings(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchOpenAIAudioTranscriptions(config, { method: 'POST', body: new FormData() }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchTags(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchShow(config, { method: 'POST', body: '{"name":"gpt-oss:120b"}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiResponses'), 'openaiResponses', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiResponsesCompact'), 'openaiResponsesCompact', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiEmbeddings'), 'openaiEmbeddings', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiAudioTranscriptions'), 'openaiAudioTranscriptions', key => key, { method: 'POST', body: new FormData() }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await ollamaFetchTags(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await ollamaFetchShow(config, { method: 'POST', body: '{"name":"gpt-oss:120b"}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -77,7 +69,7 @@ test('Authorization: Bearer is set when apiKey is configured', async () => { return new Response('{}', { status: 200 }); }, async () => { - await ollamaFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); assertEquals(auth, 'Bearer ollama_test'); @@ -95,7 +87,7 @@ test('Authorization header is omitted entirely when apiKey is absent (local daem return new Response('{}', { status: 200 }); }, async () => { - await ollamaFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); assertEquals(auth, null); @@ -110,7 +102,7 @@ test('Content-Type defaults to application/json for JSON bodies', async () => { return new Response('{}', { status: 200 }); }, async () => { - await ollamaFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); assertEquals(contentType, 'application/json'); diff --git a/packages/provider-ollama/src/account-probe.ts b/packages/provider-ollama/src/account-probe.ts index f1ddc20306..c8543cbafa 100644 --- a/packages/provider-ollama/src/account-probe.ts +++ b/packages/provider-ollama/src/account-probe.ts @@ -22,7 +22,7 @@ // and the Copilot card next to this one names its account without one. import { type OllamaUpstreamConfig } from './config.ts'; -import { ollamaFetchMe } from './fetch.ts'; +import { ollamaFetchMe } from './control-http.ts'; import { type OllamaAccountEntry, type OllamaUpstreamState, readOllamaUpstreamState } from './state.ts'; import { isOllamaUsageEnabled } from './usage-probe.ts'; import { type Fetcher, getProviderRepo, identityWrapUpstreamCall } from '@floway-dev/provider'; diff --git a/packages/provider-ollama/src/fetch.ts b/packages/provider-ollama/src/control-http.ts similarity index 61% rename from packages/provider-ollama/src/fetch.ts rename to packages/provider-ollama/src/control-http.ts index a40029abd4..d4b4cf0f36 100644 --- a/packages/provider-ollama/src/fetch.ts +++ b/packages/provider-ollama/src/control-http.ts @@ -25,28 +25,12 @@ const ollamaFetchInternal = async ( return await options.wrapUpstreamCall(() => options.fetcher(joinBaseAndPath(config.baseUrl, path), { ...init, headers })); }; -export const ollamaFetchOpenAIChatCompletions = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/chat/completions', init, options); -export const ollamaFetchOpenAIResponses = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/responses', init, options); -export const ollamaFetchOpenAIResponsesCompact = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/responses/compact', init, options); -export const ollamaFetchAnthropicMessages = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/messages', init, options); -export const ollamaFetchAnthropicMessagesCountTokens = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/messages/count_tokens', init, options); -export const ollamaFetchOpenAIEmbeddings = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/embeddings', init, options); -export const ollamaFetchOpenAICompletions = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/completions', init, options); // Stock Ollama mounts this OpenAI-compatible route, parses its multipart body, // and adapts the audio into an internal chat request. Capability discovery does // not advertise a dedicated transcription bit, so the provider exposes it only // through manual `openaiAudioTranscriptions` model entries. // https://github.com/ollama/ollama/blob/573386c35eac76124ffce571f4b0fefa0a7fe13c/server/routes.go#L1916-L1922 // https://github.com/ollama/ollama/blob/573386c35eac76124ffce571f4b0fefa0a7fe13c/middleware/openai.go#L682-L789 -export const ollamaFetchOpenAIAudioTranscriptions = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/audio/transcriptions', init, options); export const ollamaFetchTags = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => ollamaFetchInternal(config, '/api/tags', init, options); export const ollamaFetchShow = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => diff --git a/packages/provider-ollama/src/fetch-models.ts b/packages/provider-ollama/src/fetch-models.ts index f887eeef4c..5a9c0e7c3c 100644 --- a/packages/provider-ollama/src/fetch-models.ts +++ b/packages/provider-ollama/src/fetch-models.ts @@ -19,7 +19,7 @@ // /api/embed for native callers and /v1/embeddings for the OpenAI shim. import type { OllamaUpstreamConfig } from './config.ts'; -import { ollamaFetchShow, ollamaFetchTags } from './fetch.ts'; +import { ollamaFetchShow, ollamaFetchTags } from './control-http.ts'; import { discardUpstreamResponse, fetchUpstreamModels, type Fetcher, identityWrapUpstreamCall, jsonRequestBody } from '@floway-dev/provider'; export interface OllamaRawModel { diff --git a/packages/provider-ollama/src/provider.ts b/packages/provider-ollama/src/provider.ts index c7fb5f9a54..f76c51794f 100644 --- a/packages/provider-ollama/src/provider.ts +++ b/packages/provider-ollama/src/provider.ts @@ -24,25 +24,15 @@ // catalog, and an auto row carrying the same upstreamModelId is dropped so the // manual copy is the only one for that id. -import { scheduleOllamaAccountProbe } from './account-probe.ts'; import { chatFromOllamaRaw } from './chat-from-raw.ts'; -import { assertOllamaUpstreamRecord, type OllamaUpstreamConfig } from './config.ts'; +import { assertOllamaUpstreamRecord } from './config.ts'; import { OLLAMA_DEFAULT_FLAGS } from './defaults.ts'; import { fetchOllamaCatalog, type OllamaCatalog } from './fetch-models.ts'; -import { ollamaFetchOpenAIAudioTranscriptions, ollamaFetchOpenAIChatCompletions, ollamaFetchOpenAICompletions, ollamaFetchOpenAIEmbeddings, ollamaFetchAnthropicMessages, ollamaFetchAnthropicMessagesCountTokens, ollamaFetchOpenAIResponses, ollamaFetchOpenAIResponsesCompact } from './fetch.ts'; -import { pricingForOllamaModelKey } from './pricing.ts'; import { createOllamaPipelines } from './pipelines.ts'; +import { pricingForOllamaModelKey } from './pricing.ts'; import { readOllamaUpstreamState } from './state.ts'; -import { scheduleOllamaUsageProbe } from './usage-probe.ts'; -import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; -import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; -import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResult, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; -import { headersForAnthropicMessagesCall, jsonRequestBody, publicModelId, resolveEffectiveFlags, serializeModelFieldOpenAIAudioTranscriptionRequest, streamingProviderCall, type FetchInit, type FlagId, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; - -// providerData carries the raw upstream id verbatim — the same value /api/tags -// returns and the same value the gateway must send back on every inference call. -const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; +import { publicModelId, resolveEffectiveFlags, type FlagId, type ProviderInstance, type Provider, type ProviderModel, type UpstreamRecord } from '@floway-dev/provider'; // Vision / tool / thinking capabilities live alongside `embedding` in the // /api/show response. Embedding is the only one that drives a different @@ -85,27 +75,6 @@ export const createOllamaProvider = (record: UpstreamRecord): Provider => { const upstreamFlags = resolveEffectiveFlags([OLLAMA_DEFAULT_FLAGS, record.flagOverrides]); const state = readOllamaUpstreamState(record.state); - // Ollama Cloud moves the account's session and weekly windows on inference - // calls, and exposes them nowhere but its usage endpoint, so each such call - // arms a debounced background refresh. The account behind the key is armed by - // the same calls behind its own, much longer interval. Token counting never - // reaches a model and leaves the windows untouched, so it arms neither. - const armProbes = (opts: UpstreamCallOptions): void => { - scheduleOllamaUsageProbe(record.id, config, state, opts.fetcher, opts.waitUntil); - scheduleOllamaAccountProbe(record.id, config, state, opts.fetcher, opts.waitUntil); - }; - - // Arms once the upstream round-trip has produced a response, so the usage - // probe reads windows that already account for this call. A rate-limited - // response arms it too — that is exactly when an operator wants the windows - // on screen. A transport that threw never reached the account and arms - // nothing. - const withProbes = (opts: UpstreamCallOptions, dispatched: Promise): Promise => - dispatched.then(result => { - armProbes(opts); - return result; - }); - // Manual models always emit. const overriddenIds = new Set(config.models.map(m => m.upstreamModelId)); const manualModels: ProviderModel[] = config.models.map(model => { @@ -128,49 +97,7 @@ export const createOllamaProvider = (record: UpstreamRecord): Provider => { if (kind === 'chat' && model.chat) internal.chat = model.chat; return internal; }); - const call = ( - transport: (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions) => Promise, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - opts: UpstreamCallOptions, - ): Promise => { - const rawModelId = rawModelIdOf(model); - return transport( - config, - { method: 'POST', body: jsonRequestBody({ ...body, model: rawModelId }), signal }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ).then(response => ({ response, modelKey: rawModelId })); - }; - - const callStreaming = ( - transport: (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions) => Promise, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - parser: ProviderStreamParser, - opts: UpstreamCallOptions, - ) => { - const rawModelId = rawModelIdOf(model); - return streamingProviderCall( - transport( - config, - { method: 'POST', body: jsonRequestBody({ ...body, stream: true, model: rawModelId }), signal }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ), - parser, - rawModelId, - signal, - ); - }; - - const rejectUnsupported = (capability: string) => () => - Promise.reject(new Error(`Ollama provider does not implement ${capability}`)); - const instance: ProviderInstance = { - callAlphaSearch: rejectUnsupported('callAlphaSearch'), getProvidedModels: async fetcher => { const catalog = await fetchOllamaCatalog(config, fetcher); const auto = finalizeOllamaModels( @@ -179,46 +106,6 @@ export const createOllamaProvider = (record: UpstreamRecord): Provider => { ); return [...manualModels, ...auto]; }, - callOpenAICompletions: (model, body, signal, opts) => withProbes(opts, call(ollamaFetchOpenAICompletions, model, body, signal, [...opts.headers], opts)), - callOpenAIChatCompletions: (model, body, signal, opts) => withProbes(opts, callStreaming(ollamaFetchOpenAIChatCompletions, model, body, signal, [...opts.headers], parseOpenAIChatCompletionsStream, opts)), - callOpenAIResponses: async (model, body, action, signal, opts) => { - switch (action) { - case 'generate': { - const stream = await withProbes(opts, callStreaming(ollamaFetchOpenAIResponses, model, body, signal, [...opts.headers], parseOpenAIResponsesStream, opts)); - return stream.ok - ? { action: 'generate', ok: true, events: stream.events, modelKey: stream.modelKey, ...(stream.headers ? { headers: stream.headers } : {}) } - : { action: 'generate', ok: false, response: stream.response, modelKey: stream.modelKey }; - } - case 'compact': { - const rawModelId = rawModelIdOf(model); - const response = await withProbes(opts, ollamaFetchOpenAIResponsesCompact( - config, - { method: 'POST', body: jsonRequestBody({ ...toCompactPayloadShape(body), model: rawModelId }), signal }, - { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - )); - return response.ok - ? { action: 'compact', ok: true, result: (await response.json()) as OpenAIResponsesCompactionResult, modelKey: rawModelId } - : { action: 'compact', ok: false, response, modelKey: rawModelId }; - } - default: - action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${action as string}`); - } - }, - callAnthropicMessages: (model, body, signal, opts) => withProbes(opts, callStreaming(ollamaFetchAnthropicMessages, model, body, signal, headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta), parseAnthropicMessagesStream, opts)), - callAnthropicMessagesCountTokens: (model, body, signal, opts) => call(ollamaFetchAnthropicMessagesCountTokens, model, body, signal, headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta), opts), - callOpenAIEmbeddings: (model, body, signal, opts) => withProbes(opts, call(ollamaFetchOpenAIEmbeddings, model, body, signal, [...opts.headers], opts)), - // Ollama serves no image-generation endpoint; reject if the gateway ever - // routes one here. /v1/images/* is not exposed by the upstream binary. - callOpenAIImagesGenerations: rejectUnsupported('callOpenAIImagesGenerations'), - callOpenAIImagesEdits: rejectUnsupported('callOpenAIImagesEdits'), - callOpenAIAudioTranscriptions: async (model, request, signal, opts) => { - const rawModelId = rawModelIdOf(model); - const body = serializeModelFieldOpenAIAudioTranscriptionRequest(request, rawModelId); - const response = await withProbes(opts, ollamaFetchOpenAIAudioTranscriptions(config, { method: 'POST', body, signal }, { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall })); - return { response, modelKey: rawModelId }; - }, - callRerank: rejectUnsupported('callRerank'), }; return { diff --git a/packages/provider-ollama/src/usage-probe.ts b/packages/provider-ollama/src/usage-probe.ts index c555b3bc58..b6f07e7afa 100644 --- a/packages/provider-ollama/src/usage-probe.ts +++ b/packages/provider-ollama/src/usage-probe.ts @@ -31,7 +31,7 @@ // https://github.com/ollama/ollama/blob/f0078ae4766d0d570e196158f20dde309bd96124/docs/api/errors.mdx import { type OllamaUpstreamConfig } from './config.ts'; -import { ollamaFetchUsage } from './fetch.ts'; +import { ollamaFetchUsage } from './control-http.ts'; import { type OllamaUsageObservation, type OllamaUsageProbeEntry, type OllamaUpstreamState, readOllamaUpstreamState } from './state.ts'; import { type Fetcher, getProviderRepo, identityWrapUpstreamCall } from '@floway-dev/provider'; diff --git a/packages/provider/__tests__/audio_test.ts b/packages/provider/__tests__/audio_test.ts deleted file mode 100644 index 25aba03ac8..0000000000 --- a/packages/provider/__tests__/audio_test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { test } from 'vitest'; - -import { serializeModelPathOpenAIAudioTranscriptionRequest, serializeModelFieldOpenAIAudioTranscriptionRequest } from '../src/audio.ts'; -import { assertEquals, assertExists } from '@floway-dev/test-utils'; - -test('serializeModelFieldOpenAIAudioTranscriptionRequest preserves ordered fields and file metadata while replacing model', async () => { - const file = { bytes: new Uint8Array([1, 2, 3, 4]), name: 'meeting.wav', type: 'audio/wav', lastModified: 1_700_000_000_000 }; - const form = serializeModelFieldOpenAIAudioTranscriptionRequest({ - entries: [ - { name: 'file', value: file }, - { name: 'language', value: 'en' }, - { name: 'model', value: 'public-model' }, - { name: 'timestamp_granularities[]', value: 'word' }, - { name: 'timestamp_granularities[]', value: 'segment' }, - ], - }, 'upstream-model'); - - assertEquals([...form.keys()], ['file', 'language', 'model', 'timestamp_granularities[]', 'timestamp_granularities[]']); - assertEquals(form.get('model'), 'upstream-model'); - assertEquals(form.getAll('timestamp_granularities[]'), ['word', 'segment']); - const serializedFile = form.get('file'); - assertEquals(serializedFile instanceof File, true); - assertExists(serializedFile as File); - assertEquals((serializedFile as File).name, 'meeting.wav'); - assertEquals((serializedFile as File).type, 'audio/wav'); - assertEquals((serializedFile as File).lastModified, file.lastModified); - assertEquals(new Uint8Array(await (serializedFile as File).arrayBuffer()), new Uint8Array([1, 2, 3, 4])); -}); - -test('serializeModelPathOpenAIAudioTranscriptionRequest omits model fields selected by the URL', () => { - const form = serializeModelPathOpenAIAudioTranscriptionRequest({ - entries: [ - { name: 'model', value: 'public-model' }, - { name: 'file', value: { bytes: new TextEncoder().encode('audio'), name: 'meeting.wav', type: 'audio/wav' } }, - { name: 'response_format', value: 'json' }, - ], - }); - - assertEquals([...form.keys()], ['file', 'response_format']); - assertEquals(form.get('model'), null); -}); diff --git a/packages/provider/__tests__/images_test.ts b/packages/provider/__tests__/images_test.ts index e8e38848f9..adc49938c2 100644 --- a/packages/provider/__tests__/images_test.ts +++ b/packages/provider/__tests__/images_test.ts @@ -1,15 +1,26 @@ import { test } from 'vitest'; -import { serializeOpenAIImagesEditsJsonPayload, serializeOpenAIImagesEditsRequest } from '../src/images.ts'; +import { serializeOpenAIImagesEditsJsonPayload, prepareOpenAIImagesEditsBody } from '../src/images.ts'; +import { isReplayableBody } from '../src/options.ts'; +import { serializeHttpBody, type HttpMultipartBody } from '@floway-dev/http/request-content'; import { assertEquals } from '@floway-dev/test-utils'; -const parseJsonBody = async (body: Awaited>): Promise => { - if (body instanceof FormData) throw new Error('expected a JSON image-edit body'); - return await new Response(body.open()).json(); +const encodedBody = (prepared: Awaited>): Response => { + const body = serializeHttpBody(prepared.body, prepared.encoding); + if (!isReplayableBody(body)) throw new Error('Expected replayable request content'); + return new Response(body.open(), { + headers: prepared.encoding === 'multipart' + ? { 'content-type': `multipart/form-data; boundary=${(prepared.body as HttpMultipartBody).boundary}` } + : { 'content-type': 'application/json' }, + }); +}; +const parseJsonBody = async (prepared: Awaited>): Promise => { + if (prepared.encoding !== 'json') throw new Error('expected a JSON image-edit body'); + return await encodedBody(prepared).json(); }; -test('serializeOpenAIImagesEditsRequest preserves reference fields and encodes mixed uploads from their bytes', async () => { - const serialized = await serializeOpenAIImagesEditsRequest({ +test('prepareOpenAIImagesEditsBody preserves reference fields and encodes mixed uploads from their bytes', async () => { + const serialized = await prepareOpenAIImagesEditsBody({ images: [ { type: 'reference', reference: { image_url: 'https://example.test/image.png', detail: 'future-field' } }, { type: 'upload', file: { bytes: new TextEncoder().encode('inline'), name: 'inline.png', type: 'image/png' } }, @@ -42,33 +53,33 @@ test('serializeOpenAIImagesEditsJsonPayload forces upload sources into data URLs }); }); -test('serializeOpenAIImagesEditsRequest uses the singular field for one upload and the array field for many', async () => { +test('prepareOpenAIImagesEditsBody uses the singular field for one upload and the array field for many', async () => { const first = { bytes: new TextEncoder().encode('first'), name: 'first.png', type: 'image/png' }; const second = { bytes: new TextEncoder().encode('second'), name: 'second.png', type: 'image/png' }; - const single = await serializeOpenAIImagesEditsRequest({ + const single = await prepareOpenAIImagesEditsBody({ images: [{ type: 'upload', file: first }], parameters: { prompt: 'single' }, }, 'gpt-image'); - assertEquals(single instanceof FormData, true); - const singleForm = single as FormData; + assertEquals(single.encoding, 'multipart'); + const singleForm = await encodedBody(single).formData(); assertEquals((singleForm.get('image') as File).name, first.name); assertEquals(new Uint8Array(await (singleForm.get('image') as File).arrayBuffer()), first.bytes); assertEquals(singleForm.getAll('image[]'), []); - const multiple = await serializeOpenAIImagesEditsRequest({ + const multiple = await prepareOpenAIImagesEditsBody({ images: [{ type: 'upload', file: first }, { type: 'upload', file: second }], parameters: { prompt: 'multiple' }, }, 'gpt-image'); - assertEquals(multiple instanceof FormData, true); - const multipleForm = multiple as FormData; + assertEquals(multiple.encoding, 'multipart'); + const multipleForm = await encodedBody(multiple).formData(); assertEquals(multipleForm.getAll('image[]').map(value => (value as File).name), [first.name, second.name]); assertEquals(await Promise.all(multipleForm.getAll('image[]').map(async value => new Uint8Array(await (value as File).arrayBuffer()))), [first.bytes, second.bytes]); assertEquals(multipleForm.get('image'), null); assertEquals(multipleForm.get('model'), 'gpt-image'); }); -test('serializeOpenAIImagesEditsRequest leaves malformed inline data for upstream JSON validation', async () => { - const serialized = await serializeOpenAIImagesEditsRequest({ +test('prepareOpenAIImagesEditsBody leaves malformed inline data for upstream JSON validation', async () => { + const serialized = await prepareOpenAIImagesEditsBody({ images: [{ type: 'inline', reference: { image_url: 'data:image/png;base64,%%%' }, @@ -82,8 +93,8 @@ test('serializeOpenAIImagesEditsRequest leaves malformed inline data for upstrea }); }); -test('serializeOpenAIImagesEditsRequest preserves extra inline reference fields through JSON', async () => { - const serialized = await serializeOpenAIImagesEditsRequest({ +test('prepareOpenAIImagesEditsBody preserves extra inline reference fields through JSON', async () => { + const serialized = await prepareOpenAIImagesEditsBody({ images: [{ type: 'inline', reference: { image_url: 'data:image/png;base64,aW1hZ2U=', future_field: 'keep' }, diff --git a/packages/provider/__tests__/stages/decode-response_test.ts b/packages/provider/__tests__/stages/decode-response_test.ts index 57e2c7a5ba..6881c1677a 100644 --- a/packages/provider/__tests__/stages/decode-response_test.ts +++ b/packages/provider/__tests__/stages/decode-response_test.ts @@ -67,3 +67,27 @@ test('actual HTTP replies remain failure facts when a higher decoder stage throw expect(getFailureFacts(caught)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'actual-wire-model', 'response.provider.previousCalls': [] }); } }); + +test('a non2xx reply remains an unchanged HTTP exchange with no decoded stream', async () => { + const executed = await run(pipeline, entry(), services(async () => new Response('rate limited', { status: 429, headers: { 'x-upstream': 'original' } }))); + expect(executed.facts['response.provider.output']).toBeNull(); + expect(executed.facts['response.http.exchange']).toMatchObject({ type: 'response', status: 429, headers: expect.arrayContaining([['x-upstream', 'original']]) }); + expect(executed.facts['response.provider.modelKey']).toBe('actual-wire-model'); + await executed.drain(); +}); + +test.each([ + { status: 204, body: null, contentType: null, phrases: ['204', 'stream is required', 'Body: '] }, + { status: 200, body: '{"error":{"message":"azure stub"}}', contentType: 'application/json', phrases: ['200', 'application/json', 'stream is required', 'azure stub'] }, + { status: 200, body: 'not really an event stream', contentType: 'text/event-stream-fake', phrases: ['stream is required'] }, + { status: 200, body: '{"choices":[]}', contentType: null, phrases: ['"unknown"', '{"choices":[]}'] }, + { status: 200, body: 'x'.repeat(2048), contentType: 'application/json', phrases: ['...[truncated]'] }, +])('successful non-stream reply $contentType keeps diagnostic context as a failure value', async ({ status, body, contentType, phrases }) => { + const executed = await run(pipeline, entry(), services(async () => new Response(body === null ? null : new TextEncoder().encode(body), { status, headers: contentType === null ? {} : { 'content-type': contentType } }))); + const output = executed.facts['response.provider.output']; + if (output === null || 'kind' in output) throw new Error('Expected a protocol failure value'); + expect(output.status).toBe(502); + for (const phrase of phrases) expect(output.message).toContain(phrase); + expect(executed.facts['response.provider.called']).toBe(true); + await executed.drain(); +}); diff --git a/packages/provider/__tests__/streaming_test.ts b/packages/provider/__tests__/streaming_test.ts deleted file mode 100644 index 7843b9aae2..0000000000 --- a/packages/provider/__tests__/streaming_test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { test } from 'vitest'; - -import { streamingProviderCall } from '../src/streaming.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { assertEquals, assertRejects, assertStringIncludes } from '@floway-dev/test-utils'; - -interface StubEvent { type: string } - -// Stub parser: feed the body bytes through TextDecoder and yield one -// eventFrame per non-empty line, plus a terminal doneFrame. Mirrors the -// shape (but not the protocol-specific logic) of parseXxxStream so we can -// assert streamingProviderCall plumbing without dragging in protocol parsers. -const stubParser = (body: ReadableStream): AsyncIterable> => (async function* () { - const reader = body.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - for (;;) { - const { done, value } = await reader.read(); - if (done) { - buffer += decoder.decode(); - break; - } - buffer += decoder.decode(value, { stream: true }); - } - for (const line of buffer.split('\n').filter(Boolean)) { - yield eventFrame({ type: line }); - } - yield doneFrame(); -})(); - -test('streamingProviderCall returns ok:false when upstream is non-2xx', async () => { - const response = new Response('rate limited', { status: 429 }); - const result = await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - assertEquals(result.ok, false); - if (result.ok) throw new Error('expected ok:false'); - assertEquals(result.response.status, 429); - assertEquals(result.modelKey, 'm-1'); -}); - -test('streamingProviderCall throws on 2xx without a body, surfacing the status and an body marker', async () => { - // 204 is the canonical "no body" success; this is a provider-contract violation - // because the streaming endpoints always force stream:true. - const response = new Response(null, { status: 204 }); - await assertRejects(async () => { - try { - await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - } catch (error) { - assertStringIncludes((error as Error).message, '204'); - assertStringIncludes((error as Error).message, 'stream is required'); - assertStringIncludes((error as Error).message, 'Body: '); - throw error; - } - }, Error); -}); - -test('streamingProviderCall throws when 2xx content-type is not text/event-stream, including the upstream body for diagnosis', async () => { - const response = new Response(JSON.stringify({ error: { message: 'azure stub' } }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - await assertRejects(async () => { - try { - await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - } catch (error) { - assertStringIncludes((error as Error).message, '200'); - assertStringIncludes((error as Error).message, 'application/json'); - assertStringIncludes((error as Error).message, 'stream is required'); - assertStringIncludes((error as Error).message, 'azure stub'); - throw error; - } - }, Error); -}); - -test('streamingProviderCall rejects a content-type that only starts with the SSE essence', async () => { - const response = new Response('not really an event stream', { - status: 200, - headers: { 'content-type': 'text/event-stream-fake' }, - }); - await assertRejects( - () => streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined), - Error, - ); -}); - -test('streamingProviderCall surfaces "unknown" content-type when the header is missing', async () => { - // Cloudflare Workers sometimes hands us a 200 with no content-type header - // when the upstream response is malformed; the diagnostic must label that - // as "unknown" rather than the empty string so the operator can grep for it. - const response = new Response('{"choices":[]}', { status: 200 }); - await assertRejects(async () => { - try { - await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - } catch (error) { - assertStringIncludes((error as Error).message, '"unknown"'); - assertStringIncludes((error as Error).message, '{"choices":[]}'); - throw error; - } - }, Error); -}); - -test('streamingProviderCall truncates oversized bodies in the error message', async () => { - const big = 'x'.repeat(2048); - const response = new Response(big, { status: 200, headers: { 'content-type': 'application/json' } }); - await assertRejects(async () => { - try { - await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - } catch (error) { - assertStringIncludes((error as Error).message, '...[truncated]'); - throw error; - } - }, Error); -}); - -test('streamingProviderCall returns ok:true with parsed frames on 2xx SSE', async () => { - const response = new Response('alpha\nbeta\n', { - status: 200, - headers: { 'content-type': 'Text/Event-Stream; charset=utf-8' }, - }); - const result = await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - assertEquals(result.ok, true); - if (!result.ok) throw new Error('expected ok:true'); - const frames: ProtocolFrame[] = []; - for await (const frame of result.events) frames.push(frame); - assertEquals(frames, [eventFrame({ type: 'alpha' }), eventFrame({ type: 'beta' }), doneFrame()]); -}); diff --git a/packages/provider/src/audio.ts b/packages/provider/src/audio.ts index 0fa30e54d9..64a51c402f 100644 --- a/packages/provider/src/audio.ts +++ b/packages/provider/src/audio.ts @@ -5,34 +5,3 @@ export type OpenAIAudioTranscriptionFormEntry = HttpFormEntry; export interface OpenAIAudioTranscriptionRequest { readonly entries: readonly OpenAIAudioTranscriptionFormEntry[]; } - -type OpenAIAudioTranscriptionModelField = - | { readonly type: 'replace'; readonly value: string } - | { readonly type: 'omit' }; - -const serializeOpenAIAudioTranscriptionRequest = ( - request: OpenAIAudioTranscriptionRequest, - modelField: OpenAIAudioTranscriptionModelField, -): FormData => { - const form = new FormData(); - for (const entry of request.entries) { - if (entry.name === 'model') { - if (modelField.type === 'replace') form.append(entry.name, modelField.value); - } else if (typeof entry.value === 'string') { - form.append(entry.name, entry.value); - } else { - const file = entry.value; - form.append(entry.name, new File([file.bytes as Uint8Array], file.name, { type: file.type, lastModified: file.lastModified })); - } - } - return form; -}; - -export const serializeModelFieldOpenAIAudioTranscriptionRequest = ( - request: OpenAIAudioTranscriptionRequest, - model: string, -): FormData => serializeOpenAIAudioTranscriptionRequest(request, { type: 'replace', value: model }); - -export const serializeModelPathOpenAIAudioTranscriptionRequest = ( - request: OpenAIAudioTranscriptionRequest, -): FormData => serializeOpenAIAudioTranscriptionRequest(request, { type: 'omit' }); diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index c5d100a074..53a1169fa7 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -1,6 +1,4 @@ import { base64ToBytes, bytesToBase64, parseBase64ImageDataUrl } from './image-helpers.ts'; -import { jsonRequestBody } from './json-request.ts'; -import type { ReplayableBody } from './options.ts'; import { multipartBody as httpMultipartBody, type HttpFile, type HttpFormEntry, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import type { OpenAIImageEditReference } from '@floway-dev/protocols/openai-images'; @@ -96,16 +94,3 @@ export const prepareOpenAIImagesEditsBody = async (request: OpenAIImagesEditsReq ? { body: await serializeOpenAIImagesEditsJsonPayload(request, model), encoding: 'json' } : { body: httpMultipartBody(entries), encoding: 'multipart' }; }; - -export const serializeOpenAIImagesEditsRequest = async (request: OpenAIImagesEditsRequest, model: string): Promise => { - const entries = multipartEntries(request, model); - if (entries !== null) { - const form = new FormData(); - for (const { name, value } of entries) { - if (typeof value === 'string') form.append(name, value); - else form.append(name, new File([value.bytes as Uint8Array], value.name, { type: value.type, lastModified: value.lastModified })); - } - return form; - } - return jsonRequestBody(await serializeOpenAIImagesEditsJsonPayload(request, model)); -}; diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index 641e195103..b2f19fd134 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -66,11 +66,8 @@ export type { } from './provider.ts'; export { headersForAnthropicMessagesCall } from './anthropic-messages.ts'; export type { OpenAIImagesEditsRequest, OpenAIImagesEditsSource } from './images.ts'; -export { prepareOpenAIImagesEditsBody, serializeOpenAIImagesEditsJsonPayload, serializeOpenAIImagesEditsRequest } from './images.ts'; +export { prepareOpenAIImagesEditsBody, serializeOpenAIImagesEditsJsonPayload } from './images.ts'; export type { OpenAIAudioTranscriptionFormEntry, OpenAIAudioTranscriptionRequest } from './audio.ts'; -export { serializeModelFieldOpenAIAudioTranscriptionRequest, serializeModelPathOpenAIAudioTranscriptionRequest } from './audio.ts'; -export type { ProviderStreamParser } from './streaming.ts'; -export { streamingProviderCall } from './streaming.ts'; export type { ProviderRepo, UpstreamsRepoSlim } from './repo.ts'; export { getProviderRepo, initProviderRepo, UpstreamGoneError } from './repo.ts'; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index fbce3b1561..91c661cd1d 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -1,27 +1,12 @@ -import type { OpenAIAudioTranscriptionRequest } from './audio.ts'; import type { FlagDefaults } from './flags.ts'; -import type { OpenAIImagesEditsRequest } from './images.ts'; import type { ModelPrefixConfig } from './model-prefix.ts'; import type { ProviderModel, UpstreamModelsCache, UpstreamProviderKind, UpstreamRecord } from './model.ts'; import type { Fetcher } from './options.ts'; import type { ProviderPipelines } from './pipeline.ts'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { OpenAICompletionsPayload } from '@floway-dev/protocols/openai-completions'; -import type { OpenAIEmbeddingsPayload } from '@floway-dev/protocols/openai-embeddings'; -import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; +import type { OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -// Action tag threaded through the OpenAI Responses pipeline. `generate` is a normal -// streaming /responses turn; `compact` is the summarize-and-replace-history -// turn that some upstreams expose natively (`/v1/responses/compact`, -// chatgpt.com's RemoteCompactionV2 over /codex/responses) and others have to -// simulate. The same `callOpenAIResponses` method dispatches on this tag, and -// interceptors are free to flip it (the openai-responses-compact-shim turns 'compact' -// into 'generate' so the inner upstream call runs an ordinary summarization -// turn against the SUMMARIZATION_PROMPT). +// The dispatched action can change independently of the source endpoint's original intent. export type OpenAIResponsesAction = 'generate' | 'compact'; export type InboundHeaderMatcher = string | RegExp; @@ -55,33 +40,12 @@ export interface ProviderRerankCallResult extends ProviderCallResult { target: RerankTarget; } -// Streaming endpoints (Anthropic Messages / OpenAI Responses / OpenAI Chat Completions) return decoded -// protocol frames directly — the provider drives the upstream fetch, parses -// the SSE wire via @floway-dev/protocols, and emits the typed event stream. -// `ok: true` optionally carries the raw upstream `Headers` so the source-side -// `respond` layer can forward them to the downstream client (blocklist in -// gateway `shared/respond.ts` — hop-by-hop, body framing, cookies). Absent -// on lifted/synthesized streams that have no upstream Response behind them, -// matching the same shape on `EventResult`. -// `ok: false` carries the raw upstream Response verbatim so the gateway -// boundary can relay status + body + headers unchanged. Non-2xx-but-not-SSE -// responses throw from the provider as a contract violation (provider always -// forces stream=true on streaming endpoints). +// Protocol fixtures distinguish decoded frames from an HTTP failure reply. export type ProviderStreamResult = | { ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } | { ok: false; response: Response; modelKey: string }; -// `action: 'generate'` is a normal streaming /responses turn — its frames -// flow through the per-frame event stream like every other streaming endpoint. -// `action: 'compact'` is non-streaming — the upstream returns a single -// `response.compaction` envelope. Some upstreams expose a native compaction -// endpoint and produce the envelope directly; others synthesize the envelope -// from a regular /responses turn — both return the typed value rather than a -// re-parsed synthesized SSE body. The discriminated result tags which branch -// actually ran so the gateway's shape-lowering can pick between the streaming -// and value-envelope arms; snapshot mode itself reads `invocation.action` (the -// post-chain caller intent), not the result tag. -// The `ok: false` contract is identical to ProviderStreamResult above. +// The observed output shape follows the dispatched action, independently of caller intent. export type ProviderOpenAIResponsesResult = | { action: 'generate'; ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } | { action: 'generate'; ok: false; response: Response; modelKey: string } @@ -105,9 +69,8 @@ export type ProviderOpenAIResponsesResult = // The gateway filters the source request through the selected provider // instance's `inboundHeaderAllowlist` before constructing this bag. Protocol-owned // metadata is carried by its owning invocation boundary and does not widen -// this provider-level policy. A provider may clone and mutate the bag for -// request-specific wire shaping, but must not retain the gateway-owned -// reference past the call. +// this provider-level policy. Pipeline request shaping uses immutable header-line facts; this bag is the +// transport fixture and control-call boundary. export interface UpstreamCallOptions { fetcher: Fetcher; waitUntil: (promise: Promise) => void; @@ -117,7 +80,7 @@ export interface UpstreamCallOptions { // TLS, and CONNECT. Further dispatches after output preserve the measured // first-token interval, including internal server-tool continuations. // This interval includes data-plane egress and excludes model routing, translation, - // and interceptor preparation before dispatch. Candidate iteration clears + // and stage preparation before dispatch. Candidate iteration clears // both timing anchors on failover, so the recorded interval can be shorter // than the latency the client observed. wrapUpstreamCall: (dispatch: () => Promise) => Promise; @@ -131,28 +94,7 @@ export interface AnthropicMessagesUpstreamCallOptions extends UpstreamCallOption } export interface ProviderInstance { - // Catalog refresh fetches a single resource and never enters the per-request - // latency budget, so it takes the per-upstream fetcher directly instead of - // the broader `UpstreamCallOptions` bag the data-plane `call*` methods use. getProvidedModels(fetcher: Fetcher): Promise; - callAlphaSearch(model: ProviderModel, body: Record, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - // /v1/completions text completions. Passthrough. Providers whose - // upstream doesn't expose /v1/completions set `endpoints.openaiCompletions` - // to absent in getProvidedModels, so this method is unreachable for - // those upstreams; the rejecting stubs in those providers are pure - // defense-in-depth. - callOpenAICompletions(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callOpenAIChatCompletions(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise>; - callOpenAIResponses(model: ProviderModel, body: Omit, action: OpenAIResponsesAction, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callAnthropicMessages(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: AnthropicMessagesUpstreamCallOptions): Promise>; - // count_tokens is non-streaming JSON; the gateway relays the upstream - // Response verbatim. - callAnthropicMessagesCountTokens(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: AnthropicMessagesUpstreamCallOptions): Promise; - callOpenAIEmbeddings(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callOpenAIImagesGenerations(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callOpenAIImagesEdits(model: ProviderModel, request: OpenAIImagesEditsRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callOpenAIAudioTranscriptions(model: ProviderModel, request: OpenAIAudioTranscriptionRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callRerank(model: ProviderModel, request: CanonicalRerankRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; } // Static, module-shaped surface each provider package exports. The gateway @@ -163,8 +105,7 @@ export interface ProviderInstance { export interface ProviderModule { // Instance factory: capture the record and return closures. Sync — any // I/O the provider needs (token refresh, state persistence, catalog - // fetch) happens on demand inside the per-request methods on the - // returned ProviderInstance. + // fetch) happens on demand inside the operation pipelines and the catalog control callback. create: (record: UpstreamRecord) => Provider; // Exhaustive default map over every catalog flag id for a fresh // upstream of this kind; see each provider package's `defaults.ts`. diff --git a/packages/provider/src/streaming.ts b/packages/provider/src/streaming.ts deleted file mode 100644 index 0e6fcec48d..0000000000 --- a/packages/provider/src/streaming.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { ProviderStreamResult } from './provider.ts'; -import { isEventStreamMediaType, type ProtocolFrame } from '@floway-dev/protocols/common'; - -export type ProviderStreamParser = ( - body: ReadableStream, - options?: { signal?: AbortSignal }, -) => AsyncIterable>; - -const BODY_SNIPPET_CHARS = 1024; - -const readBodySnippet = async (response: Response): Promise => { - try { - const text = await response.text(); - if (text.length === 0) return ''; - return text.length > BODY_SNIPPET_CHARS ? `${text.slice(0, BODY_SNIPPET_CHARS)}...[truncated]` : text; - } catch { - return ''; - } -}; - -// A 2xx non-SSE upstream is a provider-contract violation: every streaming -// endpoint is called with stream=true. The throw bubbles to the target -// boundary, which turns it into a 502. The upstream's body (or a snippet -// of it) is folded into the error message so what the upstream actually -// returned reaches the operator instead of being discarded — content-type -// "unknown" with no body context is otherwise impossible to debug. -export const streamingProviderCall = async ( - upstreamFetch: Promise, - parser: ProviderStreamParser, - modelKey: string, - signal: AbortSignal | undefined, -): Promise> => { - const response = await upstreamFetch; - if (!response.ok) { - return { ok: false, response, modelKey }; - } - const contentType = response.headers.get('content-type') ?? ''; - if (!response.body || !isEventStreamMediaType(contentType)) { - const snippet = await readBodySnippet(response); - throw new Error(`Upstream returned ${response.status} with content-type "${contentType || 'unknown'}" but stream is required (provider must force stream=true and return text/event-stream when response.ok). Body: ${snippet}`); - } - return { ok: true, events: parser(response.body, { signal }), modelKey, headers: response.headers }; -}; diff --git a/packages/test-utils/src/chat-provider-pipeline.ts b/packages/test-utils/src/chat-provider-pipeline.ts index 5ad68f2681..845b7bf864 100644 --- a/packages/test-utils/src/chat-provider-pipeline.ts +++ b/packages/test-utils/src/chat-provider-pipeline.ts @@ -1,6 +1,6 @@ import { noopUpstreamCallOptions } from './stubs.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; -import { move, run, setRelease, type Handed } from '@floway-dev/pipeline'; +import { move, run, setRelease, type Handed, type RunServices } from '@floway-dev/pipeline'; import { providerModelFacts, type AnthropicMessagesUpstreamCallOptions, type ChatProviderOperation, type Provider, type ProviderChatResponse, type ProviderChatServices, type ProviderModel, type ProviderOperationOutputs, type ProviderOperationPayloads, type ProviderOperationRequest, type ProviderOperationResponse, type UpstreamCallOptions } from '@floway-dev/provider'; type Frame = Extract extends { frames: AsyncIterable } ? F : never; @@ -12,6 +12,7 @@ export const collectChatProviderPipeline = async = {}, ): Promise<{ facts: ProviderOperationResponse & Record; output: ProviderChatResponse['response.provider.output']; frames: Frame[]; response: Response | null }> => { const pipeline = provider.pipelines[operation]; if (pipeline === undefined) throw new Error(`${provider.kind} has no ${operation} pipeline`); @@ -22,7 +23,7 @@ export const collectChatProviderPipeline = async >; - const services: ProviderChatServices = { httpCall: () => ({ ...options, signal }), recordProtocolFrames: frames => frames }; + const services: ProviderChatServices & Pick = { ...observers, httpCall: () => ({ ...options, signal }), recordProtocolFrames: frames => frames }; const executed = await run, ProviderOperationResponse, ProviderChatServices>(pipeline, entry, services); const facts = executed.facts as ProviderOperationResponse & Record; const exchange = facts['response.http.exchange']; diff --git a/packages/test-utils/src/http-pipeline.ts b/packages/test-utils/src/http-pipeline.ts new file mode 100644 index 0000000000..3e1320aed1 --- /dev/null +++ b/packages/test-utils/src/http-pipeline.ts @@ -0,0 +1,20 @@ +import { exchangeResponse, type HttpResponseFacts, type HttpServices } from '@floway-dev/http/pipeline'; +import { move, run, setRelease, type Pipeline } from '@floway-dev/pipeline'; + +export const collectHttpPipeline = async (pipeline: Pipeline, entry: Entry, services: HttpServices): Promise<{ facts: Exit; response: Response }> => { + const executed = await run(pipeline, move(entry), services); + const exchange = executed.facts['response.http.exchange']; + try { + if (exchange.type === 'transportFailure') throw exchange.error; + const response = exchangeResponse(exchange); + const bytes = await response.arrayBuffer(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + await executed.drain(); + return { facts: executed.facts, response: new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: response.headers }) }; + } catch (error) { + try { await executed.drain(); } catch (cleanupError) { + if (cleanupError !== error) throw new AggregateError([error, cleanupError], 'HTTP test call and cleanup failed', { cause: error }); + } + throw error; + } +}; diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index 2daf3ceab5..e129c80457 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -1,3 +1,5 @@ +export { collectPreparedProviderRequest } from './prepared-provider-request.ts'; +export { collectHttpPipeline } from './http-pipeline.ts'; export { applyProviderStage, type ProviderStageProbe, type OpenAIChatCompletionsProbe, type OpenAIResponsesProbe, type AnthropicMessagesProbe } from './provider-stage.ts'; export { stubChatProviderPipeline, type StubChatProviderCall } from './stub-chat-provider-pipeline.ts'; export { collectChatProviderPipeline } from './chat-provider-pipeline.ts'; diff --git a/packages/test-utils/src/prepared-provider-request.ts b/packages/test-utils/src/prepared-provider-request.ts new file mode 100644 index 0000000000..8c1c1779dd --- /dev/null +++ b/packages/test-utils/src/prepared-provider-request.ts @@ -0,0 +1,28 @@ +import { collectHttpPipeline } from './http-pipeline.ts'; +import { stubProviderModel } from './stubs.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose, type Stage } from '@floway-dev/pipeline'; +import { observeProviderCall, providerModelFacts, type FetchInit, type ProviderOperation, type ProviderRequest, type ProviderResponse, type UpstreamFetchOptions } from '@floway-dev/provider'; + +export const collectPreparedProviderRequest = async (prepare: Stage, operation: ProviderOperation, modelData: (modelKey: string) => unknown, init: FetchInit, options: UpstreamFetchOptions, selectedModel?: string): Promise => { + const fields: Record = typeof init.body === 'string' ? JSON.parse(init.body) as Record : {}; + const modelKey = selectedModel ?? (typeof fields.model === 'string' ? fields.model : 'test-model'); + const { model: _, ...json } = fields; + let payload: object = json; + if (init.body instanceof FormData) { + const entries = await Promise.all([...init.body].map(async ([name, value]) => ({ name, value: typeof value === 'string' ? value : { name: value.name, type: value.type, lastModified: value.lastModified, bytes: new Uint8Array(await value.arrayBuffer()) } }))); + payload = { entries }; + } + const headers = [...new Headers(init.headers), ...options.extraHeaders ?? []].map(([name, value]): [string, string] => [name, value]); + const beta = new Headers(headers).get('anthropic-beta'); + const pipeline = compose, ProviderResponse>('testPreparedProviderRequest', [prepare, observeProviderCall, http]); + const result = await collectHttpPipeline(pipeline, { + 'request.provider.model': providerModelFacts(stubProviderModel({ id: modelKey, providerData: modelData(modelKey) })), + 'request.provider.payload': payload, + 'request.http.callId': 0, + 'request.http.headers': headers, + ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? { 'request.provider.responsesAction': operation === 'openaiResponsesCompact' ? 'compact' : 'generate' } : {}), + ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? { 'request.provider.anthropicBeta': beta === null ? [] : beta.split(',') } : {}), + }, { httpCall: () => ({ ...options, signal: init.signal ?? undefined, waitUntil: () => {} }) }); + return result.response; +}; diff --git a/packages/test-utils/src/stubs.ts b/packages/test-utils/src/stubs.ts index 7b2fd727a3..8321e1e4ba 100644 --- a/packages/test-utils/src/stubs.ts +++ b/packages/test-utils/src/stubs.ts @@ -1,11 +1,7 @@ import { testFetcher } from './mock-fetch.ts'; import { type FlagId, type InternalModel, type PerformanceTelemetryContext, type ProviderInstance, type Provider, type ProviderModel, type ModelCandidate, type TelemetryModelIdentity, type AnthropicMessagesUpstreamCallOptions, type UpstreamCallOptions, identityWrapUpstreamCall } from '@floway-dev/provider'; -// No-op UpstreamCallOptions factory for tests calling provider methods -// directly: the fetcher uses runtime fetch so `globalThis.fetch` spies still -// intercept; waitUntil drops the promise (the runtime would have absorbed it -// in production). The Headers bag is per-call so tests that mutate it do not -// bleed state across cases. +// The runtime fetch spy remains visible behind replayable bodies, and each fixture owns its headers. export const noopUpstreamCallOptions = (overrides: Partial = {}): UpstreamCallOptions => ({ fetcher: testFetcher, waitUntil: () => {}, @@ -20,10 +16,7 @@ export const noopAnthropicMessagesUpstreamCallOptions = (overrides: Partial = {}): ProviderModel => ({ id: 'test-model', upstreamModelId: 'test-model', @@ -78,24 +71,13 @@ export const mockPerfTelemetryContext = (overrides: Partial = {}): ProviderInstance => ({ getProvidedModels: overrides.getProvidedModels ?? (() => Promise.resolve([])), - callAlphaSearch: overrides.callAlphaSearch ?? (() => Promise.reject(new Error('stubProvider.callAlphaSearch was called'))), - callOpenAICompletions: overrides.callOpenAICompletions ?? (() => Promise.reject(new Error('stubProvider.callOpenAICompletions was called'))), - callOpenAIChatCompletions: overrides.callOpenAIChatCompletions ?? (() => Promise.reject(new Error('stubProvider.callOpenAIChatCompletions was called'))), - callOpenAIResponses: overrides.callOpenAIResponses ?? (() => Promise.reject(new Error('stubProvider.callOpenAIResponses was called'))), - callAnthropicMessages: overrides.callAnthropicMessages ?? (() => Promise.reject(new Error('stubProvider.callAnthropicMessages was called'))), - callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens ?? (() => Promise.reject(new Error('stubProvider.callAnthropicMessagesCountTokens was called'))), - callOpenAIEmbeddings: overrides.callOpenAIEmbeddings ?? (() => Promise.reject(new Error('stubProvider.callOpenAIEmbeddings was called'))), - callOpenAIImagesGenerations: overrides.callOpenAIImagesGenerations ?? (() => Promise.reject(new Error('stubProvider.callOpenAIImagesGenerations was called'))), - callOpenAIImagesEdits: overrides.callOpenAIImagesEdits ?? (() => Promise.reject(new Error('stubProvider.callOpenAIImagesEdits was called'))), - callOpenAIAudioTranscriptions: overrides.callOpenAIAudioTranscriptions ?? (() => Promise.reject(new Error('stubProvider.callOpenAIAudioTranscriptions was called'))), - callRerank: overrides.callRerank ?? (() => Promise.reject(new Error('stubProvider.callRerank was called'))), }); // Stitches together a candidate whose `model.providerModels` map carries an // entry under the wired provider's upstream id — that's what // `providerModelOf(candidate)` resolves to at dispatch time. The // `enabledFlags` / `providerData` shortcuts populate that ProviderModel -// directly — the common case for interceptor tests that just need a flag set +// directly — the common case for stage tests that just need a flag set // for the resolver's own upstream key. Any `model.providerModels` supplied // through `overrides.model` replaces both the default entry and those // shortcuts wholesale. From c9f7f6f33537d32b04c7cd033d205cc3e87ec752 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:49:52 +0800 Subject: [PATCH 128/156] refactor(provider): finish subscription operation pipelines Remove legacy business call surfaces and interceptor drivers after routing Codex and Claude Code through provider-owned facts and stages. Keep pure request preparation, state observation and protocol restoration in current modules. Prepare Codex content before token refresh and retain it across retries. Move existing fixtures onto actual stage and operation chains; verify scoped stream diagnostics and production session identity ordering. --- .../__tests__/fetch_test.ts | 34 +- .../__tests__/provider_test.ts | 32 +- .../backfill-required-fields_test.ts | 24 +- .../hoist-user-system-to-messages_test.ts | 20 +- .../inject-billing-block_test.ts | 18 +- .../inject-default-template_test.ts | 22 +- .../inject-identity-block_test.ts | 18 +- .../synthesize-metadata-user-id_test.ts | 47 +- .../system-blocks_test.ts | 2 +- .../__tests__/test-utils/call.ts | 24 + packages/provider-claude-code/package.json | 1 - .../provider-claude-code/src/access-token.ts | 2 +- packages/provider-claude-code/src/backend.ts | 242 ++++++++ packages/provider-claude-code/src/fetch.ts | 526 ------------------ packages/provider-claude-code/src/headers.ts | 2 +- packages/provider-claude-code/src/index.ts | 4 +- .../backfill-required-fields.ts | 36 -- .../hoist-user-system-to-messages.ts | 67 --- .../interceptors/anthropic-messages/index.ts | 55 -- .../inject-billing-block.ts | 32 -- .../inject-default-template.ts | 80 --- .../inject-identity-block.ts | 18 - .../synthesize-metadata-user-id.ts | 83 --- .../interceptors/anthropic-messages/types.ts | 14 - packages/provider-claude-code/src/models.ts | 2 +- packages/provider-claude-code/src/provider.ts | 70 --- .../src/stages/authenticate.ts | 2 +- .../src/stages/backfill-required-fields.ts | 31 +- .../src/stages/decode-response.ts | 2 +- .../stages/hoist-user-system-to-messages.ts | 59 +- .../src/stages/inject-billing-block.ts | 25 +- .../src/stages/inject-default-template.ts | 72 ++- .../src/stages/inject-identity-block.ts | 10 +- .../src/stages/observe-response.ts | 2 +- .../src/stages/prepare-request.ts | 2 +- .../src/stages/retry-access.ts | 2 +- .../src/stages/synthesize-metadata-user-id.ts | 75 ++- .../anthropic-messages => }/system-blocks.ts | 0 .../provider-codex/__tests__/fetch_test.ts | 86 ++- .../provider-codex/__tests__/provider_test.ts | 50 +- .../action-pivot_test.ts | 60 +- .../inject-default-instructions_test.ts | 19 +- ...strip-unsupported-response-fields_test.ts} | 15 +- .../__tests__/test-utils/call.ts | 36 ++ packages/provider-codex/package.json | 1 - .../src/{fetch.ts => backend.ts} | 481 +--------------- .../interceptors/openai-responses/index.ts | 21 - .../inject-default-instructions.ts | 20 - .../strip-unsupported-fields.ts | 32 -- .../interceptors/openai-responses/types.ts | 17 - packages/provider-codex/src/models.ts | 2 +- packages/provider-codex/src/pipeline-facts.ts | 2 +- packages/provider-codex/src/pipelines.ts | 7 +- packages/provider-codex/src/provider.ts | 80 +-- .../provider-codex/src/stages/authenticate.ts | 2 +- .../src/stages/inject-default-instructions.ts | 13 +- .../src/stages/observe-response.ts | 2 +- .../src/stages/prepare-responses-content.ts | 15 + .../src/stages/prepare-responses.ts | 20 +- .../provider-codex/src/stages/retry-access.ts | 2 +- .../strip-unsupported-response-fields.ts | 25 +- pnpm-lock.yaml | 6 - 62 files changed, 869 insertions(+), 1902 deletions(-) rename packages/provider-claude-code/__tests__/{interceptors/anthropic-messages => stages}/backfill-required-fields_test.ts (76%) rename packages/provider-claude-code/__tests__/{interceptors/anthropic-messages => stages}/hoist-user-system-to-messages_test.ts (72%) rename packages/provider-claude-code/__tests__/{interceptors/anthropic-messages => stages}/inject-billing-block_test.ts (73%) rename packages/provider-claude-code/__tests__/{interceptors/anthropic-messages => stages}/inject-default-template_test.ts (83%) rename packages/provider-claude-code/__tests__/{interceptors/anthropic-messages => stages}/inject-identity-block_test.ts (64%) rename packages/provider-claude-code/__tests__/{interceptors/anthropic-messages => stages}/synthesize-metadata-user-id_test.ts (70%) rename packages/provider-claude-code/__tests__/{interceptors/anthropic-messages => }/system-blocks_test.ts (98%) create mode 100644 packages/provider-claude-code/__tests__/test-utils/call.ts create mode 100644 packages/provider-claude-code/src/backend.ts delete mode 100644 packages/provider-claude-code/src/fetch.ts delete mode 100644 packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts delete mode 100644 packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts delete mode 100644 packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts delete mode 100644 packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts delete mode 100644 packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts delete mode 100644 packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts delete mode 100644 packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts delete mode 100644 packages/provider-claude-code/src/interceptors/anthropic-messages/types.ts rename packages/provider-claude-code/src/{interceptors/anthropic-messages => }/system-blocks.ts (100%) rename packages/provider-codex/__tests__/{interceptors/openai-responses => stages}/action-pivot_test.ts (67%) rename packages/provider-codex/__tests__/{interceptors/openai-responses => stages}/inject-default-instructions_test.ts (80%) rename packages/provider-codex/__tests__/{interceptors/openai-responses/strip-unsupported-fields_test.ts => stages/strip-unsupported-response-fields_test.ts} (85%) create mode 100644 packages/provider-codex/__tests__/test-utils/call.ts rename packages/provider-codex/src/{fetch.ts => backend.ts} (50%) delete mode 100644 packages/provider-codex/src/interceptors/openai-responses/index.ts delete mode 100644 packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts delete mode 100644 packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts delete mode 100644 packages/provider-codex/src/interceptors/openai-responses/types.ts create mode 100644 packages/provider-codex/src/stages/prepare-responses-content.ts diff --git a/packages/provider-claude-code/__tests__/fetch_test.ts b/packages/provider-claude-code/__tests__/fetch_test.ts index 9d8161faea..79c6f9bec0 100644 --- a/packages/provider-claude-code/__tests__/fetch_test.ts +++ b/packages/provider-claude-code/__tests__/fetch_test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; -import { callClaudeCodeAnthropicMessages } from '../src/fetch.ts'; +import { callClaudeCodeAnthropicMessages } from './test-utils/call.ts'; import { CLAUDE_CODE_HEADERS_HAIKU, CLAUDE_CODE_HEADERS_SONNET_OPUS } from '../src/headers.ts'; import type { ClaudeCodeAccessTokenEntry, @@ -84,8 +84,6 @@ const seedAccount = (overrides: Partial = {}): void const readQuotaEntry = (): ClaudeCodeQuotaSnapshotEntry | null => currentState().accounts[0]!.quotaSnapshot; -// Yield to the queue so the .catch chain from fireAndForgetPersist completes -// before assertions inspect the persisted state. const flushAsyncQueue = (): Promise => new Promise(resolve => setTimeout(resolve, 0)); beforeEach(() => { @@ -304,7 +302,7 @@ describe('callClaudeCodeAnthropicMessages — wire body', () => { describe('callClaudeCodeAnthropicMessages — incomplete stream diagnostics', () => { test('logs upstream trace headers and last raw SSE frames when message_stop is absent', async () => { seedAccount({ accessToken: freshAccessTokenEntry }); - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const warn = vi.fn(); vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response( new ReadableStream({ start(controller) { @@ -324,22 +322,18 @@ describe('callClaudeCodeAnthropicMessages — incomplete stream diagnostics', () )); const result = await callClaudeCodeAnthropicMessages({ - upstreamId, model: sonnetModel, body: minimalBody, shaped: false, call: noopUpstreamCallOptions(), + upstreamId, model: sonnetModel, body: minimalBody, shaped: false, call: noopUpstreamCallOptions(), observers: { log: { debug: () => {}, info: () => {}, warn, error: () => {} } }, }); expect(result.ok).toBe(true); if (!result.ok) return; await drain(result.events); expect(warn).toHaveBeenCalledTimes(1); - const line = warn.mock.calls[0]![0] as string; - expect(line).toContain('claude_code_messages_stream_incomplete'); - expect(line).toContain('upstream_id=up_cc'); - expect(line).toContain('request_id=req_trace'); - expect(line).toContain('cf_ray=ray_trace'); - expect(line).toContain('trace_response=trace_response'); - expect(line).toContain('raw_sse_frames=2'); - expect(line).toContain('terminal_event=null'); - expect(line).toContain('message_delta'); + expect(warn.mock.calls[0]![0]).toBe('claude_code_messages_stream_incomplete'); + const fields = warn.mock.calls[0]![1] as Record; + expect(fields).toMatchObject({ upstream_id: upstreamId, request_id: 'req_trace', cf_ray: 'ray_trace', trace_response: 'trace_response', raw_sse_frames: 2, terminal_event: null }); + expect(fields.last_sse_frames).toContain('message_delta'); + }); test('does not log a complete Messages stream', async () => { @@ -444,12 +438,7 @@ describe('callClaudeCodeAnthropicMessages — quota persistence', () => { expect(data.status).toBe('rejected'); }); - test('2xx → registers persist promise via opts.call.waitUntil exactly once', async () => { - // On Cloudflare Workers the runtime cancels orphan promises the moment - // the response is sent. The gateway threads `waitUntil` through - // UpstreamCallOptions so the persist can extend the worker's lifetime; - // assert we hand the persist promise to it exactly once and that the - // promise actually mutates state when awaited. + test('2xx quota write completes through the run deferred lifecycle', async () => { seedAccount({ accessToken: freshAccessTokenEntry }); vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); const waitUntil = vi.fn<(promise: Promise) => void>(); @@ -458,10 +447,7 @@ describe('callClaudeCodeAnthropicMessages — quota persistence', () => { upstreamId, model: sonnetModel, body: minimalBody, shaped: false, call, }); expect(result.ok).toBe(true); - expect(waitUntil).toHaveBeenCalledTimes(1); - const handed = waitUntil.mock.calls[0]![0]; - expect(handed).toBeInstanceOf(Promise); - await expect(handed).resolves.toBeUndefined(); + expect(waitUntil).not.toHaveBeenCalled(); const stored = readQuotaEntry(); expect(stored).not.toBeNull(); }); diff --git a/packages/provider-claude-code/__tests__/provider_test.ts b/packages/provider-claude-code/__tests__/provider_test.ts index 7afcfb14ff..a989493256 100644 --- a/packages/provider-claude-code/__tests__/provider_test.ts +++ b/packages/provider-claude-code/__tests__/provider_test.ts @@ -175,7 +175,7 @@ describe('createClaudeCodeProvider — factory surface', () => { }); }); -describe('createClaudeCodeProvider — callAnthropicMessages routes through chain', () => { +describe('createClaudeCodeProvider — Messages operation stages', () => { test.each([false, true])('the actual Messages pipeline preserves shaped=%s content and dated model identity', async shaped => { const provider = createClaudeCodeProvider(currentRecord); const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); @@ -199,6 +199,21 @@ describe('createClaudeCodeProvider — callAnthropicMessages routes through chai expect(new Headers(init.headers).get('anthropic-beta')).toEqual(shaped ? 'oauth-2025-04-20' : expect.stringContaining('claude-code-20250219')); }); + test('production session identity derives from the original user turn before system hoisting', async () => { + const provider = createClaudeCodeProvider(currentRecord); + const bodies: WireAnthropicMessagesPayload[] = []; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => { + bodies.push(await readJsonRequest(init!) as WireAnthropicMessagesPayload); + return sseResponse(); + }); + for (const prompt of ['What is the capital of France?', 'Who wrote The Great Gatsby?']) { + await collectChatProviderPipeline(provider, 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, system: 'Shared system instructions', messages: [{ role: 'user', content: prompt }] }, undefined, noopAnthropicMessagesUpstreamCallOptions()); + } + const ids = bodies.map(body => (JSON.parse(body.metadata.user_id) as { session_id: string }).session_id); + expect(ids[0]).not.toBe(ids[1]); + expect(bodies.every(body => JSON.stringify(body.messages[0]).includes('Shared system instructions'))).toBe(true); + }); + test('the Messages pipeline preserves a complete terminal refusal while recording state persistence', async () => { const body = { error: { type: 'permission_error', message: 'OAuth authentication is currently not allowed for this organization' }, diagnostic: 'x'.repeat(800) }; vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(body, { status: 403, headers: { 'x-upstream': 'retained' } })); @@ -215,12 +230,11 @@ describe('createClaudeCodeProvider — callAnthropicMessages routes through chai const instance = createClaudeCodeProvider(currentRecord); const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); - await instance.instance.callAnthropicMessages( + await collectChatProviderPipeline(instance, 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, - noopAnthropicMessagesUpstreamCallOptions(), - ); + noopAnthropicMessagesUpstreamCallOptions()); const init = fetchSpy.mock.calls[0]![1] as RequestInit; const wireHeaders = new Headers(init.headers); @@ -242,7 +256,7 @@ describe('createClaudeCodeProvider — callAnthropicMessages routes through chai const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); const userId = JSON.stringify({ device_id: 'd'.repeat(32), account_uuid: '', session_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' }); - await instance.instance.callAnthropicMessages( + await collectChatProviderPipeline(instance, 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, @@ -251,8 +265,7 @@ describe('createClaudeCodeProvider — callAnthropicMessages routes through chai metadata: { user_id: userId }, }, undefined, - cliClientCallOpts(), - ); + cliClientCallOpts()); const init = fetchSpy.mock.calls[0]![1] as RequestInit; const body = await readJsonRequest(init) as WireAnthropicMessagesPayload; @@ -277,12 +290,11 @@ describe('createClaudeCodeProvider — callAnthropicMessages routes through chai const instance = createClaudeCodeProvider(currentRecord); const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); - await instance.instance.callAnthropicMessages( + await collectChatProviderPipeline(instance, 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, - { ...noopAnthropicMessagesUpstreamCallOptions(), headers: new Headers({ 'user-agent': 'claude-cli/2.1.181' }) }, - ); + { ...noopAnthropicMessagesUpstreamCallOptions(), headers: new Headers({ 'user-agent': 'claude-cli/2.1.181' }) }); const init = fetchSpy.mock.calls[0]![1] as RequestInit; const body = await readJsonRequest(init) as WireAnthropicMessagesPayload; diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts b/packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts similarity index 76% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts rename to packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts index 93192c87d9..370d934e1c 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts +++ b/packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts @@ -1,16 +1,20 @@ import { test } from 'vitest'; -import { backfillRequiredFields } from '../../../src/interceptors/anthropic-messages/backfill-required-fields.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { backfillRequiredFields } from '../../src/stages/backfill-required-fields.ts'; import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderModel, ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload, model: ProviderModel = stubProviderModel({ endpoints: { anthropicMessages: {} } })): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload, model: ProviderModel = stubProviderModel({ endpoints: { anthropicMessages: {} } })): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model, upstreamId: 'up_test', }); @@ -26,7 +30,7 @@ test('backfills max_tokens from ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS when both temperature: 0.5, } as Partial as AnthropicMessagesPayload); - await backfillRequiredFields(ctx, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.max_tokens, ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS); }); @@ -42,7 +46,7 @@ test('prefers model.limits.max_output_tokens over the gateway fallback when set' model, ); - await backfillRequiredFields(ctx, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.max_tokens, 64000); }); @@ -59,7 +63,7 @@ test('preserves caller-supplied max_tokens', async () => { model, ); - await backfillRequiredFields(ctx, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.max_tokens, 100); }); @@ -74,7 +78,7 @@ test('backfills temperature to 1 when missing', async () => { messages: [{ role: 'user', content: 'hi' }], }); - await backfillRequiredFields(ctx, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.temperature, 1); }); @@ -87,7 +91,7 @@ test('preserves caller-supplied temperature', async () => { temperature: 0.7, }); - await backfillRequiredFields(ctx, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.temperature, 0.7); }); @@ -102,7 +106,7 @@ test('preserves caller-supplied temperature: 0', async () => { temperature: 0, }); - await backfillRequiredFields(ctx, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.temperature, 0); }); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts b/packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts similarity index 72% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts rename to packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts index f2b68e00ce..216ca69e52 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts +++ b/packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts @@ -1,16 +1,20 @@ import { test } from 'vitest'; -import { hoistUserSystemToMessages } from '../../../src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { hoistUserSystemToMessages } from '../../src/stages/hoist-user-system-to-messages.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId: 'up_test', }); @@ -23,7 +27,7 @@ test('captures a string system into a synthetic user/assistant pair and drops `s system: 'You are a pirate.', }); - await hoistUserSystemToMessages(ctx, okEvents); + await applyProviderStage(hoistUserSystemToMessages, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages, [ @@ -44,7 +48,7 @@ test('joins multi-block system into one synthetic turn with blank-line separator ], }); - await hoistUserSystemToMessages(ctx, okEvents); + await applyProviderStage(hoistUserSystemToMessages, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages[0], { role: 'user', content: [{ type: 'text', text: '[System Instructions]\nfirst rule\n\nsecond rule' }] }); @@ -57,7 +61,7 @@ test('drops system entirely when caller did not send one and leaves messages unt messages: [{ role: 'user', content: 'hi' }], }); - await hoistUserSystemToMessages(ctx, okEvents); + await applyProviderStage(hoistUserSystemToMessages, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages.length, 1); @@ -72,7 +76,7 @@ test('drops system but does not inject a synthetic turn when system text is empt system: [{ type: 'text', text: '' }], }); - await hoistUserSystemToMessages(ctx, okEvents); + await applyProviderStage(hoistUserSystemToMessages, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages.length, 1); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts b/packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts similarity index 73% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts rename to packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts index b564732440..9c3150ce75 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts @@ -1,17 +1,21 @@ import { test } from 'vitest'; -import { CLAUDE_CLI_VERSION } from '../../../src/headers.ts'; -import { injectBillingBlock } from '../../../src/interceptors/anthropic-messages/inject-billing-block.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { CLAUDE_CLI_VERSION } from '../../src/headers.ts'; +import { injectBillingBlock } from '../../src/stages/inject-billing-block.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId: 'up_test', }); @@ -23,7 +27,7 @@ test('drops a single billing block as system[0] with the pinned CLI version and messages: [{ role: 'user', content: 'hello world' }], }); - await injectBillingBlock(ctx, okEvents); + await applyProviderStage(injectBillingBlock, ctx, okEvents, { 'request.claudeCode.shaped': false }); const system = ctx.payload.system; if (!Array.isArray(system)) throw new Error('expected system to be an array'); @@ -46,7 +50,7 @@ test('overwrites any pre-existing system array (hoist already ran)', async () => system: [{ type: 'text', text: 'stale leftover' } satisfies AnthropicMessagesTextBlock], }); - await injectBillingBlock(ctx, okEvents); + await applyProviderStage(injectBillingBlock, ctx, okEvents, { 'request.claudeCode.shaped': false }); const system = ctx.payload.system; if (!Array.isArray(system)) throw new Error('expected system to be an array'); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts b/packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts similarity index 83% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts rename to packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts index 706abcfab0..6e748674d4 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts @@ -1,17 +1,21 @@ import { test } from 'vitest'; -import { injectDefaultTemplate } from '../../../src/interceptors/anthropic-messages/inject-default-template.ts'; -import { DEFAULT_TEMPLATE_BLOCK, IDENTITY_BLOCK } from '../../../src/interceptors/anthropic-messages/system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { injectDefaultTemplate } from '../../src/stages/inject-default-template.ts'; +import { DEFAULT_TEMPLATE_BLOCK, IDENTITY_BLOCK } from '../../src/system-blocks.ts'; import type { AnthropicMessagesClientTool, AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId: 'up_test', }); @@ -26,7 +30,7 @@ test('appends DEFAULT_TEMPLATE_BLOCK as system[2] with ephemeral cache_control i system: [billingBlock, IDENTITY_BLOCK], }); - await injectDefaultTemplate(ctx, okEvents); + await applyProviderStage(injectDefaultTemplate, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, [billingBlock, IDENTITY_BLOCK, DEFAULT_TEMPLATE_BLOCK]); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); @@ -57,7 +61,7 @@ test('preserves ephemeral cache_control when caller already holds 3 breakpoints tools: [cachedTool], }); - await injectDefaultTemplate(ctx, okEvents); + await applyProviderStage(injectDefaultTemplate, ctx, okEvents, { 'request.claudeCode.shaped': false }); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); assertEquals(ctx.payload.system.length, 4); @@ -94,7 +98,7 @@ test('demotes our cache_control when caller already holds 4 breakpoints (would b tools: [cachedTool], }); - await injectDefaultTemplate(ctx, okEvents); + await applyProviderStage(injectDefaultTemplate, ctx, okEvents, { 'request.claudeCode.shaped': false }); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); assertEquals(ctx.payload.system.length, 4); @@ -124,7 +128,7 @@ test('demotes our cache_control when caller already exceeds the cap', async () = tools: [cachedTool], }); - await injectDefaultTemplate(ctx, okEvents); + await applyProviderStage(injectDefaultTemplate, ctx, okEvents, { 'request.claudeCode.shaped': false }); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); const injected = ctx.payload.system.at(-1)!; diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts b/packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts similarity index 64% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts rename to packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts index 494c39567d..38ffb1dff4 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts @@ -1,17 +1,21 @@ import { test } from 'vitest'; -import { injectIdentityBlock } from '../../../src/interceptors/anthropic-messages/inject-identity-block.ts'; -import { IDENTITY_BLOCK } from '../../../src/interceptors/anthropic-messages/system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { injectIdentityBlock } from '../../src/stages/inject-identity-block.ts'; +import { IDENTITY_BLOCK } from '../../src/system-blocks.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId: 'up_test', }); @@ -25,7 +29,7 @@ test('appends IDENTITY_BLOCK after an existing system[0] block', async () => { system: [billing], }); - await injectIdentityBlock(ctx, okEvents); + await applyProviderStage(injectIdentityBlock, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, [billing, IDENTITY_BLOCK]); }); @@ -39,7 +43,7 @@ test('appends IDENTITY_BLOCK onto a one-block system array regardless of block c system: [existing], }); - await injectIdentityBlock(ctx, okEvents); + await applyProviderStage(injectIdentityBlock, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, [existing, IDENTITY_BLOCK]); }); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts b/packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts similarity index 70% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts rename to packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts index e65ccdf8f5..964cb648b8 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts +++ b/packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts @@ -1,20 +1,23 @@ import { parse, validate, version } from 'uuid'; import { test } from 'vitest'; -import { parseMetadataUserID } from '../../../src/detection.ts'; -import { hoistUserSystemToMessages } from '../../../src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts'; -import { CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY } from '../../../src/interceptors/anthropic-messages/index.ts'; -import { synthesizeMetadataUserId } from '../../../src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { parseMetadataUserID } from '../../src/detection.ts'; +import { hoistUserSystemToMessages } from '../../src/stages/hoist-user-system-to-messages.ts'; +import { synthesizeMetadataUserId } from '../../src/stages/synthesize-metadata-user-id.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload, upstreamId = 'up_test'): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload, upstreamId = 'up_test'): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId, }); @@ -26,7 +29,7 @@ test('fills metadata.user_id with the new JSON shape when absent', async () => { messages: [{ role: 'user', content: 'hello' }], }); - await synthesizeMetadataUserId(ctx, okEvents); + await applyProviderStage(synthesizeMetadataUserId(ctx.upstreamId), ctx, okEvents, { 'request.claudeCode.shaped': false }); const userId = ctx.payload.metadata?.user_id; if (typeof userId !== 'string') throw new Error('expected user_id to be a string'); @@ -43,8 +46,8 @@ test('fills metadata.user_id with the new JSON shape when absent', async () => { test('device_id is stable per upstream id', async () => { const a = invocation({ model: 'claude-sonnet-4-5-20250929', max_tokens: 1, messages: [{ role: 'user', content: 'a' }] }); const b = invocation({ model: 'claude-sonnet-4-5-20250929', max_tokens: 1, messages: [{ role: 'user', content: 'b' }] }); - await synthesizeMetadataUserId(a, okEvents); - await synthesizeMetadataUserId(b, okEvents); + await applyProviderStage(synthesizeMetadataUserId(a.upstreamId), a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(synthesizeMetadataUserId(b.upstreamId), b, okEvents, { 'request.claudeCode.shaped': false }); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; assertEquals(ad.deviceId, bd.deviceId); @@ -53,8 +56,8 @@ test('device_id is stable per upstream id', async () => { test('device_id differs across upstreams', async () => { const a = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'x' }] }, 'up_a'); const b = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'x' }] }, 'up_b'); - await synthesizeMetadataUserId(a, okEvents); - await synthesizeMetadataUserId(b, okEvents); + await applyProviderStage(synthesizeMetadataUserId(a.upstreamId), a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(synthesizeMetadataUserId(b.upstreamId), b, okEvents, { 'request.claudeCode.shaped': false }); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; if (ad.deviceId === bd.deviceId) throw new Error('expected different device ids per upstream'); @@ -63,8 +66,8 @@ test('device_id differs across upstreams', async () => { test('session_id is stable for same upstream + same first-user prefix', async () => { const a = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'prefix' }, { role: 'assistant', content: 'reply1' }] }); const b = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'prefix' }, { role: 'assistant', content: 'reply2' }] }); - await synthesizeMetadataUserId(a, okEvents); - await synthesizeMetadataUserId(b, okEvents); + await applyProviderStage(synthesizeMetadataUserId(a.upstreamId), a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(synthesizeMetadataUserId(b.upstreamId), b, okEvents, { 'request.claudeCode.shaped': false }); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; assertEquals(ad.sessionId, bd.sessionId); @@ -78,7 +81,7 @@ test('preserves a caller-supplied user_id verbatim', async () => { messages: [{ role: 'user', content: 'x' }], metadata: { user_id: explicit }, }); - await synthesizeMetadataUserId(ctx, okEvents); + await applyProviderStage(synthesizeMetadataUserId(ctx.upstreamId), ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.metadata?.user_id, explicit); }); @@ -105,8 +108,10 @@ test('session_id differs when system prompt is shared but user message differs ( // Drive the same step pair the production chain does: synthesize first, // then hoist. Synthesize sees the operator's real first user message; // hoist runs after and rewrites `messages` for the wire shape. - await synthesizeMetadataUserId(a, () => hoistUserSystemToMessages(a, okEvents)); - await synthesizeMetadataUserId(b, () => hoistUserSystemToMessages(b, okEvents)); + await applyProviderStage(synthesizeMetadataUserId(a.upstreamId), a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(hoistUserSystemToMessages, a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(synthesizeMetadataUserId(b.upstreamId), b, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(hoistUserSystemToMessages, b, okEvents, { 'request.claudeCode.shaped': false }); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; @@ -114,11 +119,3 @@ test('session_id differs when system prompt is shared but user message differs ( throw new Error('expected different session_ids for distinct user prompts sharing a system prompt'); } }); - -test('chain registers synthesize before hoist', () => { - const chain = CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY; - const synthIdx = chain.indexOf(synthesizeMetadataUserId); - const hoistIdx = chain.indexOf(hoistUserSystemToMessages); - if (synthIdx === -1 || hoistIdx === -1) throw new Error('chain missing required step'); - if (synthIdx >= hoistIdx) throw new Error(`synthesize (${synthIdx}) must run before hoist (${hoistIdx}) so session_id derives from the real user message`); -}); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/system-blocks_test.ts b/packages/provider-claude-code/__tests__/system-blocks_test.ts similarity index 98% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/system-blocks_test.ts rename to packages/provider-claude-code/__tests__/system-blocks_test.ts index 6642ddd92e..e9428118e4 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/system-blocks_test.ts +++ b/packages/provider-claude-code/__tests__/system-blocks_test.ts @@ -5,7 +5,7 @@ import { computeCcVersionFingerprint, DEFAULT_TEMPLATE_BLOCK, IDENTITY_BLOCK, -} from '../../../src/interceptors/anthropic-messages/system-blocks.ts'; +} from '../src/system-blocks.ts'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; const minimalBody = (firstUserText: string): AnthropicMessagesPayload => ({ diff --git a/packages/provider-claude-code/__tests__/test-utils/call.ts b/packages/provider-claude-code/__tests__/test-utils/call.ts new file mode 100644 index 0000000000..7b19e17638 --- /dev/null +++ b/packages/provider-claude-code/__tests__/test-utils/call.ts @@ -0,0 +1,24 @@ +import { authenticateClaudeCode } from '../../src/stages/authenticate.ts'; +import { decodeClaudeCodeResponse } from '../../src/stages/decode-response.ts'; +import { observeClaudeCodeResponse } from '../../src/stages/observe-response.ts'; +import { prepareClaudeCodeRequest } from '../../src/stages/prepare-request.ts'; +import { retryClaudeCodeAccess } from '../../src/stages/retry-access.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose, defineStage, move, type RunServices } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { observeProviderCall, type Provider, type ProviderModel, type ProviderStreamResult, type AnthropicMessagesUpstreamCallOptions } from '@floway-dev/provider'; +import { collectChatProviderPipeline } from '@floway-dev/test-utils'; + +interface Options { upstreamId: string; model: ProviderModel; body: Omit; shaped: boolean; signal?: AbortSignal; call: AnthropicMessagesUpstreamCallOptions; observers?: Pick } +export const callClaudeCodeAnthropicMessages = async (opts: Options): Promise> => { + const prepare = defineStage, Record, Record, Record>({ + name: 'supplyClaudeCodePreparedFixture', + through: { request: { needs: [], consumes: [], provides: ['request.claudeCode.shaped', 'request.provider.modelKey'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.claudeCode.shaped': opts.shaped, 'request.provider.modelKey': (opts.model.providerData as { upstreamModelId: string }).upstreamModelId })) }), + }); + const provider: Provider = { upstreamId: opts.upstreamId, kind: 'claude-code', name: 'Claude Code fixture', inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: { getProvidedModels: async () => [] }, pipelines: { anthropicMessages: compose('claudeCode.transportFixture', [prepare, authenticateClaudeCode(opts.upstreamId), prepareClaudeCodeRequest, decodeClaudeCodeResponse(opts.upstreamId), retryClaudeCodeAccess(opts.upstreamId), observeClaudeCodeResponse(opts.upstreamId), observeProviderCall, http]) } }; + const result = await collectChatProviderPipeline(provider, 'anthropicMessages', opts.model, opts.body, opts.signal, opts.call, opts.observers); + if (result.output !== null && 'kind' in result.output && result.output.kind === 'stream') return { ok: true, modelKey: result.facts['response.provider.modelKey'], events: (async function* () { yield* result.frames; })() }; + if (result.response === null) throw new Error('Expected Messages stream or HTTP failure'); + return { ok: false, modelKey: result.facts['response.provider.modelKey'], response: result.response }; +}; diff --git a/packages/provider-claude-code/package.json b/packages/provider-claude-code/package.json index af071f58d5..27124730a1 100644 --- a/packages/provider-claude-code/package.json +++ b/packages/provider-claude-code/package.json @@ -12,7 +12,6 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", "@noble/hashes": "^2.2.0", diff --git a/packages/provider-claude-code/src/access-token.ts b/packages/provider-claude-code/src/access-token.ts index 91145cb0c9..a02946fa56 100644 --- a/packages/provider-claude-code/src/access-token.ts +++ b/packages/provider-claude-code/src/access-token.ts @@ -198,7 +198,7 @@ const ensureClaudeCodeAccessTokenInner = async ( return { entry: newAccessTokenEntry, freshlyMinted: true }; }; -// Terminal flip from the oauth-error path. Distinct from fetch.ts's +// Terminal flip from the oauth-error path. Distinct from backend.ts's // `persistTerminalAccountState`: the caller has already read the account and // established that it is active, so the log's identity and `from_state` come // from that read instead of a second one, and the trigger is an oauth-protocol diff --git a/packages/provider-claude-code/src/backend.ts b/packages/provider-claude-code/src/backend.ts new file mode 100644 index 0000000000..047725942a --- /dev/null +++ b/packages/provider-claude-code/src/backend.ts @@ -0,0 +1,242 @@ +import { logWarn, logInfo, type LogFields } from './log.ts'; +import { type ClaudeCodeQuotaSnapshot } from './quota.ts'; +import { + readClaudeCodeUpstreamState, + replaceSoleAccount, + type ClaudeCodeAccountCredential, +} from './state.ts'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { getProviderRepo } from '@floway-dev/provider'; + +export const ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages?beta=true'; +export const STREAM_DIAGNOSTIC_FRAME_LIMIT = 3; +export const STREAM_DIAGNOSTIC_FRAME_DATA_CHARS = 256; + +export const synthetic503 = (message: string): Response => + new Response( + JSON.stringify({ error: { type: 'claude_code_upstream_unavailable', message } }), + { status: 503, headers: { 'content-type': 'application/json' } }, + ); + +export const synthetic429 = (message: string, retryAtIso: string | null, now: Date): Response => { + const retryAfterSeconds = retryAtIso === null + ? 60 + : Math.max(0, Math.ceil((new Date(retryAtIso).getTime() - now.getTime()) / 1000)); + return new Response( + JSON.stringify({ error: { type: 'claude_code_rate_limited', message, retry_at: retryAtIso } }), + { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': String(retryAfterSeconds) }, + }, + ); +}; + +export interface StreamDiagnosticFrame { + event: string | null; + data: string; +} + +export const observedClaudeCodeMessagesStream = async function* ( + events: AsyncIterable>, + options: { + upstreamId: string; + model: string; + headers: Headers; + signal: AbortSignal | undefined; + frames: StreamDiagnosticFrame[]; + rawFrameCount: () => number; + warn: (message: string, fields: Readonly>) => Promise; + }, +): AsyncGenerator> { + let terminalEvent: 'message_stop' | 'error' | null = null; + let streamError: unknown; + try { + for await (const frame of events) { + if (frame.type === 'event') { + if (frame.event.type === 'message_stop') terminalEvent = 'message_stop'; + else if (frame.event.type === 'error') terminalEvent = 'error'; + } + yield frame; + } + } catch (error) { + streamError = error; + throw error; + } finally { + if (!options.signal?.aborted && !(terminalEvent !== null && streamError === undefined)) { + const fields = { + upstream_id: options.upstreamId, + model: options.model, + request_id: options.headers.get('request-id'), + cf_ray: options.headers.get('cf-ray'), + trace_response: options.headers.get('traceresponse'), + raw_sse_frames: options.rawFrameCount(), + terminal_event: terminalEvent, + error: streamError === undefined ? null : streamError, + last_sse_frames: JSON.stringify(options.frames), + }; + try { await options.warn('claude_code_messages_stream_incomplete', fields); } catch (recordingError) { + if (streamError === undefined || recordingError === streamError) throw recordingError; + throw new AggregateError([streamError, recordingError], 'Claude Code stream failed and recording also failed', { cause: streamError }); + } + } + } +}; + +// `anthropic-ratelimit-unified-status: rejected` paired with a future +// `unified-reset` timestamp means the upstream's primary plan window is +// exhausted and a fresh request would 429 right away; short-circuit at +// the gate so we don't burn an OAuth refresh on a request that has no +// chance. +// +// Note 1: `overage.status: rejected` (typically paired with +// `overage-disabled-reason: out_of_credits`) is NOT a short-circuit +// signal. It only reports that the account has no extra-usage credits +// to spill into once the primary window runs out — which is the steady +// state for any plan-tier account that hasn't bought extra credits, so +// blocking on it would refuse every request to such accounts. The +// primary `status` already reflects whether the upstream will actually +// reject the next request. +// +// Note 2: a primary `status: rejected` WITHOUT a `reset` is treated as +// non-gating. Sub2api `ratelimit_service.go:953-961` flags this exact +// shape as "likely not a real rate limit" (e.g. an "Extra usage required" +// body sentinel) and passes it through verbatim — without a reset we'd +// otherwise lock the account out indefinitely because the next request +// never fires to refresh the snapshot. +export const isRateLimitedNow = ( + snapshot: ClaudeCodeQuotaSnapshot | null, + now: Date, +): snapshot is ClaudeCodeQuotaSnapshot => { + if (!snapshot) return false; + if (snapshot.status !== 'rejected') return false; + if (!snapshot.reset) return false; + return new Date(snapshot.reset).getTime() > now.getTime(); +}; + +export const persistQuotaSnapshot = async (upstreamId: string, snapshot: ClaudeCodeQuotaSnapshot, info: (message: string, fields: LogFields) => void | Promise = logInfo): Promise => { + // Stamped before the write: the mutator is replayed on a lost race and must + // return the same document each time, and this records when the snapshot was + // observed rather than which attempt landed it. + const fetchedAt = Date.now(); + // The prior status comes from the state the mutator was handed: the write is + // retried against whoever won the row, so only that view describes the + // snapshot this write actually replaced. + let previousAccount!: ClaudeCodeAccountCredential; + await getProviderRepo().upstreams.saveState(upstreamId, current => { + const state = readClaudeCodeUpstreamState(current); + previousAccount = state.accounts[0]; + return replaceSoleAccount(state, account => ({ + ...account, + quotaSnapshot: { fetchedAt, data: snapshot }, + })); + }); + const priorStatus = previousAccount.quotaSnapshot === null ? null : previousAccount.quotaSnapshot.data.status; + // Emit only on transition. Persisting every response would flood the log + // with one event per request; the dashboard already reads the snapshot + // verbatim. Operators care about the moment the upstream flipped from + // `allowed` to `rejected` (or back), not the steady state. + if (priorStatus !== snapshot.status) { + await info('claude_code_quota_state_transition', { + upstream_id: upstreamId, + account_uuid: previousAccount.accountUuid, + from_status: priorStatus, + to_status: snapshot.status, + reset_at_iso: snapshot.reset, + representative_claim: snapshot.representativeClaim, + }); + } +}; + +// Credential-class terminal sentinels on the data-plane response body. Both +// signal a permanently disabled org that no retry, refresh, or re-import +// can recover — the operator must contact Anthropic. Matching the +// lowercased `error.message` substring mirrors sub2api +// `ratelimit_service.go:208-214` (400 path) and CRS +// `claudeRelayService.js:140-153` (`_isOrganizationDisabledError`, both +// 400 and 403). We match on "organization has been disabled" rather than +// CRS's slightly longer "this organization has been disabled" so a body +// that omits the leading "this" still matches (sub2api uses the shorter +// form too). +const ORG_DISABLED_400_SENTINEL = 'organization has been disabled'; +const ORG_BANNED_403_SENTINEL = 'oauth authentication is currently not allowed'; + +interface AnthropicErrorBody { + error?: { type?: unknown; message?: unknown }; +} + +// Returns the operator-facing terminal message when the response matches +// one of the credential-class sentinels, or `null` otherwise. The body +// parse is intentionally defensive: a 400/403 that isn't JSON (or whose +// JSON shape doesn't match `{error:{type,message}}`) is the common case +// for unrelated errors (`max_tokens` validation, beta-feature gating, +// etc.) and must NOT trigger a terminal flip. +export const detectTerminalSentinel = (status: number, bodyText: string): string | null => { + if (status !== 400 && status !== 403) return null; + let parsed: unknown; + try { + parsed = JSON.parse(bodyText); + } catch { + logWarn('claude_code_unparseable_error_body', { status, body_snippet: bodyText.slice(0, 256) }); + return null; + } + if (typeof parsed !== 'object' || parsed === null) return null; + const error = (parsed as AnthropicErrorBody).error; + if (typeof error !== 'object' || error === null) return null; + const type = error.type; + const message = error.message; + if (typeof type !== 'string' || typeof message !== 'string') return null; + const lowered = message.toLowerCase(); + if (status === 400 && type === 'invalid_request_error' && lowered.includes(ORG_DISABLED_400_SENTINEL)) { + return `Organization disabled by Anthropic — re-import will not recover; contact support: ${message}`; + } + if (status === 403 && type === 'permission_error' && lowered.includes(ORG_BANNED_403_SENTINEL)) { + return `Organization banned from OAuth by Anthropic — re-import will not recover; contact support: ${message}`; + } + return null; +}; + +// Terminal flip from the data-plane sentinel detector. Distinct from +// access-token.ts's `persistTerminalState`: this path runs in a +// deferred response-observation context, so the account it logs +// comes from the mutator's own view; the flip is body-sentinel-triggered (org +// disabled/banned), not oauth-error-triggered, so the log carries +// `upstream_status` instead of `oauth_code`; and a sibling oauth-side flip may +// have already happened, so an account that is no longer `active` is returned +// untouched — the repo reads that as "nothing to do" and the dashboard keeps +// the first signal. Merging the two helpers would force conditional dispatch +// on every one of these axes. +export const persistTerminalAccountState = async ( + upstreamId: string, + terminalMessage: string, + reason: string, + upstreamStatus: number, + warn: (message: string, fields: LogFields) => void | Promise = logWarn, +): Promise => { + // Stamped before the write for the same reason as the quota snapshot: a + // replay must produce the same document. + const flippedAt = new Date().toISOString(); + let previousAccount!: ClaudeCodeAccountCredential; + await getProviderRepo().upstreams.saveState(upstreamId, current => { + const state = readClaudeCodeUpstreamState(current); + previousAccount = state.accounts[0]; + if (previousAccount.state !== 'active') return state; + return replaceSoleAccount(state, account => ({ + ...account, + state: 'refresh_failed', + stateMessage: terminalMessage, + stateUpdatedAt: flippedAt, + accessToken: null, + })); + }); + if (previousAccount.state !== 'active') return; + await warn('claude_code_account_state_flip', { + upstream_id: upstreamId, + account_uuid: previousAccount.accountUuid, + from_state: previousAccount.state, + to_state: 'refresh_failed', + reason, + upstream_status: upstreamStatus, + message: terminalMessage, + }); +}; diff --git a/packages/provider-claude-code/src/fetch.ts b/packages/provider-claude-code/src/fetch.ts deleted file mode 100644 index 7e76ef231c..0000000000 --- a/packages/provider-claude-code/src/fetch.ts +++ /dev/null @@ -1,526 +0,0 @@ -import { ensureClaudeCodeAccessToken, invalidateClaudeCodeAccessToken, type EnsuredAccessToken } from './access-token.ts'; -import { ClaudeCodeOAuthSessionTerminatedError } from './auth/oauth.ts'; -import { pickClaudeCodeHeaders } from './headers.ts'; -import { logWarn, logInfo, type LogFields } from './log.ts'; -import type { ClaudeCodeProviderData } from './models.ts'; -import { parseClaudeCodeQuotaHeaders, type ClaudeCodeQuotaSnapshot } from './quota.ts'; -import { - readClaudeCodeUpstreamState, - replaceSoleAccount, - type ClaudeCodeAccountCredential, -} from './state.ts'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; -import { - getProviderRepo, - headersForAnthropicMessagesCall, - jsonRequestBody, - streamingProviderCall, - type AnthropicMessagesUpstreamCallOptions, - type ProviderModel, - type ProviderStreamResult, -} from '@floway-dev/provider'; - -export const ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages?beta=true'; -export const STREAM_DIAGNOSTIC_FRAME_LIMIT = 3; -export const STREAM_DIAGNOSTIC_FRAME_DATA_CHARS = 256; - -export interface CallClaudeCodeAnthropicMessagesOptions { - upstreamId: string; - model: ProviderModel; - body: Omit; - // `shaped: true` means the inbound request already looks like real CC - // traffic. The gateway has reduced ordinary headers to the provider module's - // allowlist and carries anthropic-beta as typed Anthropic Messages metadata, so the - // wire path preserves the complete genuine fingerprint. It supplies a - // default Content-Type when absent and sets cached OAuth auth. - // `shaped: false` means the gateway's re-mimicry chain rebuilt the - // payload's system blocks / metadata / model id — replace headers with - // the pinned CC set so the wire shape matches end-to-end. - shaped: boolean; - signal?: AbortSignal; - call: AnthropicMessagesUpstreamCallOptions; -} - -export const synthetic503 = (message: string): Response => - new Response( - JSON.stringify({ error: { type: 'claude_code_upstream_unavailable', message } }), - { status: 503, headers: { 'content-type': 'application/json' } }, - ); - -export const synthetic429 = (message: string, retryAtIso: string | null, now: Date): Response => { - const retryAfterSeconds = retryAtIso === null - ? 60 - : Math.max(0, Math.ceil((new Date(retryAtIso).getTime() - now.getTime()) / 1000)); - return new Response( - JSON.stringify({ error: { type: 'claude_code_rate_limited', message, retry_at: retryAtIso } }), - { - status: 429, - headers: { 'content-type': 'application/json', 'retry-after': String(retryAfterSeconds) }, - }, - ); -}; - -const oneLineError = (error: unknown): string => { - const message = (error instanceof Error ? error.message : String(error)).replace(/\s+/g, ' ').trim(); - return message.length > 512 ? `${message.slice(0, 509)}...` : message; -}; - -export interface StreamDiagnosticFrame { - event: string | null; - data: string; -} - -export const observedClaudeCodeMessagesStream = async function* ( - events: AsyncIterable>, - options: { - upstreamId: string; - model: string; - headers: Headers; - signal: AbortSignal | undefined; - frames: StreamDiagnosticFrame[]; - rawFrameCount: () => number; - warn?: (message: string, fields: Readonly>) => Promise; - }, -): AsyncGenerator> { - let terminalEvent: 'message_stop' | 'error' | null = null; - let streamError: unknown; - try { - for await (const frame of events) { - if (frame.type === 'event') { - if (frame.event.type === 'message_stop') terminalEvent = 'message_stop'; - else if (frame.event.type === 'error') terminalEvent = 'error'; - } - yield frame; - } - } catch (error) { - streamError = error; - throw error; - } finally { - if (!options.signal?.aborted && !(terminalEvent !== null && streamError === undefined)) { - const fields = { - upstream_id: options.upstreamId, - model: options.model, - request_id: options.headers.get('request-id'), - cf_ray: options.headers.get('cf-ray'), - trace_response: options.headers.get('traceresponse'), - raw_sse_frames: options.rawFrameCount(), - terminal_event: terminalEvent, - error: streamError === undefined ? null : options.warn === undefined ? oneLineError(streamError) : streamError, - last_sse_frames: JSON.stringify(options.frames), - }; - if (options.warn === undefined) logWarn('claude_code_messages_stream_incomplete', { ...fields, error: streamError === undefined ? null : oneLineError(streamError) }); - else { - try { await options.warn('claude_code_messages_stream_incomplete', fields); } catch (recordingError) { - if (streamError === undefined || recordingError === streamError) throw recordingError; - throw new AggregateError([streamError, recordingError], 'Claude Code stream failed and recording also failed', { cause: streamError }); - } - } - } - } -}; - -// `anthropic-ratelimit-unified-status: rejected` paired with a future -// `unified-reset` timestamp means the upstream's primary plan window is -// exhausted and a fresh request would 429 right away; short-circuit at -// the gate so we don't burn an OAuth refresh on a request that has no -// chance. -// -// Note 1: `overage.status: rejected` (typically paired with -// `overage-disabled-reason: out_of_credits`) is NOT a short-circuit -// signal. It only reports that the account has no extra-usage credits -// to spill into once the primary window runs out — which is the steady -// state for any plan-tier account that hasn't bought extra credits, so -// blocking on it would refuse every request to such accounts. The -// primary `status` already reflects whether the upstream will actually -// reject the next request. -// -// Note 2: a primary `status: rejected` WITHOUT a `reset` is treated as -// non-gating. Sub2api `ratelimit_service.go:953-961` flags this exact -// shape as "likely not a real rate limit" (e.g. an "Extra usage required" -// body sentinel) and passes it through verbatim — without a reset we'd -// otherwise lock the account out indefinitely because the next request -// never fires to refresh the snapshot. -export const isRateLimitedNow = ( - snapshot: ClaudeCodeQuotaSnapshot | null, - now: Date, -): snapshot is ClaudeCodeQuotaSnapshot => { - if (!snapshot) return false; - if (snapshot.status !== 'rejected') return false; - if (!snapshot.reset) return false; - return new Date(snapshot.reset).getTime() > now.getTime(); -}; - -export const persistQuotaSnapshot = async (upstreamId: string, snapshot: ClaudeCodeQuotaSnapshot, info: (message: string, fields: LogFields) => void | Promise = logInfo): Promise => { - // Stamped before the write: the mutator is replayed on a lost race and must - // return the same document each time, and this records when the snapshot was - // observed rather than which attempt landed it. - const fetchedAt = Date.now(); - // The prior status comes from the state the mutator was handed: the write is - // retried against whoever won the row, so only that view describes the - // snapshot this write actually replaced. - let previousAccount!: ClaudeCodeAccountCredential; - await getProviderRepo().upstreams.saveState(upstreamId, current => { - const state = readClaudeCodeUpstreamState(current); - previousAccount = state.accounts[0]; - return replaceSoleAccount(state, account => ({ - ...account, - quotaSnapshot: { fetchedAt, data: snapshot }, - })); - }); - const priorStatus = previousAccount.quotaSnapshot === null ? null : previousAccount.quotaSnapshot.data.status; - // Emit only on transition. Persisting every response would flood the log - // with one event per request; the dashboard already reads the snapshot - // verbatim. Operators care about the moment the upstream flipped from - // `allowed` to `rejected` (or back), not the steady state. - if (priorStatus !== snapshot.status) { - await info('claude_code_quota_state_transition', { - upstream_id: upstreamId, - account_uuid: previousAccount.accountUuid, - from_status: priorStatus, - to_status: snapshot.status, - reset_at_iso: snapshot.reset, - representative_claim: snapshot.representativeClaim, - }); - } -}; - -// Best-effort persist: the hot path must not block on the write completing or -// surface its failures to the caller. Skip writing when the response carries -// no rate-limit signal at all — that would erase the prior snapshot for no -// upside. -// -// On Cloudflare Workers the runtime cancels orphan promises the moment the -// response is sent to the client, so a bare fire-and-forget would lose the -// write on the hot path. The `waitUntil` callback (when supplied by the -// gateway) extends the worker's lifetime past the response so the persist -// completes. When `waitUntil` is undefined (Node target / tests), the -// promise still runs to completion under the host event loop -// — and tests can observe it by awaiting a microtask flush. -const persistQuotaFromHeadersFireAndForget = ( - upstreamId: string, - headers: Headers, - waitUntil: ((promise: Promise) => void) | undefined, -): void => { - const snapshot = parseClaudeCodeQuotaHeaders(headers); - if (Object.keys(snapshot.raw).length === 0) return; - const persist = persistQuotaSnapshot(upstreamId, snapshot).catch(error => { - logWarn('claude_code_quota_persist_failed', { - upstream_id: upstreamId, - error: String(error), - }); - }); - waitUntil?.(persist); -}; - -// Credential-class terminal sentinels on the data-plane response body. Both -// signal a permanently disabled org that no retry, refresh, or re-import -// can recover — the operator must contact Anthropic. Matching the -// lowercased `error.message` substring mirrors sub2api -// `ratelimit_service.go:208-214` (400 path) and CRS -// `claudeRelayService.js:140-153` (`_isOrganizationDisabledError`, both -// 400 and 403). We match on "organization has been disabled" rather than -// CRS's slightly longer "this organization has been disabled" so a body -// that omits the leading "this" still matches (sub2api uses the shorter -// form too). -const ORG_DISABLED_400_SENTINEL = 'organization has been disabled'; -const ORG_BANNED_403_SENTINEL = 'oauth authentication is currently not allowed'; - -interface AnthropicErrorBody { - error?: { type?: unknown; message?: unknown }; -} - -// Returns the operator-facing terminal message when the response matches -// one of the credential-class sentinels, or `null` otherwise. The body -// parse is intentionally defensive: a 400/403 that isn't JSON (or whose -// JSON shape doesn't match `{error:{type,message}}`) is the common case -// for unrelated errors (`max_tokens` validation, beta-feature gating, -// etc.) and must NOT trigger a terminal flip. -export const detectTerminalSentinel = (status: number, bodyText: string): string | null => { - if (status !== 400 && status !== 403) return null; - let parsed: unknown; - try { - parsed = JSON.parse(bodyText); - } catch { - logWarn('claude_code_unparseable_error_body', { status, body_snippet: bodyText.slice(0, 256) }); - return null; - } - if (typeof parsed !== 'object' || parsed === null) return null; - const error = (parsed as AnthropicErrorBody).error; - if (typeof error !== 'object' || error === null) return null; - const type = error.type; - const message = error.message; - if (typeof type !== 'string' || typeof message !== 'string') return null; - const lowered = message.toLowerCase(); - if (status === 400 && type === 'invalid_request_error' && lowered.includes(ORG_DISABLED_400_SENTINEL)) { - return `Organization disabled by Anthropic — re-import will not recover; contact support: ${message}`; - } - if (status === 403 && type === 'permission_error' && lowered.includes(ORG_BANNED_403_SENTINEL)) { - return `Organization banned from OAuth by Anthropic — re-import will not recover; contact support: ${message}`; - } - return null; -}; - -// Terminal flip from the data-plane sentinel detector. Distinct from -// access-token.ts's `persistTerminalState`: this path runs in a -// fire-and-forget context with no caller-side state, so the account it logs -// comes from the mutator's own view; the flip is body-sentinel-triggered (org -// disabled/banned), not oauth-error-triggered, so the log carries -// `upstream_status` instead of `oauth_code`; and a sibling oauth-side flip may -// have already happened, so an account that is no longer `active` is returned -// untouched — the repo reads that as "nothing to do" and the dashboard keeps -// the first signal. Merging the two helpers would force conditional dispatch -// on every one of these axes. -export const persistTerminalAccountState = async ( - upstreamId: string, - terminalMessage: string, - reason: string, - upstreamStatus: number, - warn: (message: string, fields: LogFields) => void | Promise = logWarn, -): Promise => { - // Stamped before the write for the same reason as the quota snapshot: a - // replay must produce the same document. - const flippedAt = new Date().toISOString(); - let previousAccount!: ClaudeCodeAccountCredential; - await getProviderRepo().upstreams.saveState(upstreamId, current => { - const state = readClaudeCodeUpstreamState(current); - previousAccount = state.accounts[0]; - if (previousAccount.state !== 'active') return state; - return replaceSoleAccount(state, account => ({ - ...account, - state: 'refresh_failed', - stateMessage: terminalMessage, - stateUpdatedAt: flippedAt, - accessToken: null, - })); - }); - if (previousAccount.state !== 'active') return; - await warn('claude_code_account_state_flip', { - upstream_id: upstreamId, - account_uuid: previousAccount.accountUuid, - from_state: previousAccount.state, - to_state: 'refresh_failed', - reason, - upstream_status: upstreamStatus, - message: terminalMessage, - }); -}; - -// Fire-and-forget: clones the response so the original body still streams -// to the caller intact (we surface upstream verbatim — the terminal flip -// is purely additional dashboard signal). -// -// Same lifecycle dance as `persistQuotaFromHeadersFireAndForget`: under -// Workers the runtime cancels orphan promises the moment the response is -// sent, so we register the persist with `waitUntil` when the gateway -// supplied it. -const maybePersistTerminalFromBodyFireAndForget = ( - upstreamId: string, - response: Response, - waitUntil: ((promise: Promise) => void) | undefined, -): void => { - if (response.status !== 400 && response.status !== 403) return; - const cloned = response.clone(); - const task = (async () => { - let bodyText: string; - try { - bodyText = await cloned.text(); - } catch (error) { - logWarn('claude_code_terminal_sentinel_body_read_failed', { - upstream_id: upstreamId, - upstream_status: response.status, - error: String(error), - }); - return; - } - const terminalMessage = detectTerminalSentinel(response.status, bodyText); - if (terminalMessage === null) return; - const reason = response.status === 400 ? 'org_disabled_400_sentinel' : 'org_banned_403_sentinel'; - logWarn('claude_code_terminal_sentinel_detected', { - upstream_id: upstreamId, - upstream_status: response.status, - reason, - }); - await persistTerminalAccountState(upstreamId, terminalMessage, reason, response.status); - })().catch(error => { - logWarn('claude_code_terminal_sentinel_persist_failed', { - upstream_id: upstreamId, - error: String(error), - }); - }); - waitUntil?.(task); -}; - -const syntheticReturn = ( - upstreamModelId: string, - response: Response, -): ProviderStreamResult => ({ - ok: false, - modelKey: upstreamModelId, - response, -}); - -// Either ensures a usable access token or returns a 503 wrap for terminal -// refresh failures; other errors propagate. Used at both the cold-start -// call site and the 401-retry branch so the catch shape lives in one place. -const ensureOrSession503 = async ( - opts: CallClaudeCodeAnthropicMessagesOptions, - upstreamModelId: string, -): Promise> => { - try { - return await ensureClaudeCodeAccessToken({ - upstreamId: opts.upstreamId, - repo: getProviderRepo().upstreams, - fetcher: opts.call.fetcher, - }); - } catch (err) { - if (err instanceof ClaudeCodeOAuthSessionTerminatedError) { - // ensureClaudeCodeAccessToken already persisted the terminal state. - return syntheticReturn(upstreamModelId, synthetic503(`Claude Code refresh failed: ${err.upstreamMessage}`)); - } - throw err; - } -}; - -export const callClaudeCodeAnthropicMessages = async ( - opts: CallClaudeCodeAnthropicMessagesOptions, -): Promise> => { - // `opts.model.id` is the public alias on the catalog; the dated upstream id - // Anthropic expects on the wire — and that the pricing table keys by — rides - // on `opts.model.providerData.upstreamModelId`. Resolve once so synthetic - // gates, the wire body, and the streaming-call modelKey all surface the - // same dated id. - const upstreamModelId = (opts.model.providerData as ClaudeCodeProviderData).upstreamModelId; - - const fresh = await getProviderRepo().upstreams.getById(opts.upstreamId); - if (!fresh) throw new Error(`Claude Code upstream ${opts.upstreamId} disappeared mid-request`); - const state = readClaudeCodeUpstreamState(fresh.state); - const account = state.accounts[0]; - - if (account.state !== 'active') { - return syntheticReturn(upstreamModelId, synthetic503( - `Claude Code account is ${account.state}: ${account.stateMessage}`, - )); - } - - const now = new Date(); - const quotaData = account.quotaSnapshot === null ? null : account.quotaSnapshot.data; - if (isRateLimitedNow(quotaData, now)) { - const resetIso = quotaData.reset; - return syntheticReturn(upstreamModelId, synthetic429( - resetIso ? `Claude Code upstream rate-limited until ${resetIso}` : 'Claude Code upstream rate-limited', - resetIso, - now, - )); - } - - const ensured = await ensureOrSession503(opts, upstreamModelId); - if ('modelKey' in ensured) return ensured; - - return await performUpstreamCall(opts, upstreamModelId, ensured, false); -}; - -const performUpstreamCall = async ( - opts: CallClaudeCodeAnthropicMessagesOptions, - upstreamModelId: string, - accessToken: EnsuredAccessToken, - alreadyRetried: boolean, -): Promise> => { - let responseHeaders: Headers | undefined; - let rawSseFrameCount = 0; - const rawSseFrames: StreamDiagnosticFrame[] = []; - const observeRawSseFrame = (frame: SseFrame): void => { - rawSseFrameCount += 1; - rawSseFrames.push({ - event: frame.event ?? null, - data: frame.data.length > STREAM_DIAGNOSTIC_FRAME_DATA_CHARS - ? `${frame.data.slice(0, STREAM_DIAGNOSTIC_FRAME_DATA_CHARS - 3)}...` - : frame.data, - }); - if (rawSseFrames.length > STREAM_DIAGNOSTIC_FRAME_LIMIT) rawSseFrames.shift(); - }; - let headers: Record; - if (opts.shaped) { - // The gateway already reduced ordinary headers to the Claude Code module's - // allowlist. This path restores typed Anthropic Messages metadata, preserves the - // resulting fingerprint, fills Content-Type when absent, and sets - // provider-owned OAuth auth. - const passthrough = Object.fromEntries(headersForAnthropicMessagesCall([...opts.call.headers], opts.call.anthropicBeta)); - // Sub2api always sets Content-Type when the inbound omits it - // (`gateway_service.go` request-forwarding path), so the upstream - // never receives a body-bearing request without a media type. - if (!('content-type' in passthrough)) passthrough['content-type'] = 'application/json'; - headers = { ...passthrough, authorization: `Bearer ${accessToken.entry.token}` }; - } else { - headers = { ...pickClaudeCodeHeaders(upstreamModelId), authorization: `Bearer ${accessToken.entry.token}` }; - } - - // Force stream:true regardless of caller intent. The streaming envelope is - // what the gateway boundary expects; non-streaming Anthropic Messages is routed - // elsewhere. Safe in the shaped passthrough path too: shaped detection - // requires CC client headers + system blocks + a valid metadata.user_id, - // and the real Claude Code client always sets `stream: true`. - const wireBody: AnthropicMessagesPayload = { ...opts.body, model: upstreamModelId, stream: true }; - - const upstreamFetch = opts.call.wrapUpstreamCall(() => opts.call.fetcher(ANTHROPIC_MESSAGES_ENDPOINT, { - method: 'POST', - headers, - body: jsonRequestBody(wireBody), - signal: opts.signal, - })).then(response => { - responseHeaders = response.headers; - // `opts.call.waitUntil` is set by the gateway on Workers so the - // runtime keeps the worker alive past the response (without it, the - // persist promise gets cancelled the moment the response returns). - // Undefined under hosts that don't supply it (Node target / tests). - const { waitUntil } = opts.call; - // Every Anthropic response (2xx or 429) ships an - // `anthropic-ratelimit-unified-*` snapshot; capture both so the rate- - // limited gate above stays accurate as the window evolves. Other - // statuses (4xx/5xx outside 429) carry no quota signal so we skip them. - if (response.ok || response.status === 429) { - persistQuotaFromHeadersFireAndForget(opts.upstreamId, response.headers, waitUntil); - } - // 400 / 403 may carry the credential-class terminal sentinels — the - // detector is body-shape-defensive and only flips on a real match, so - // unrelated 400s (`max_tokens` validation, etc.) pass straight through. - maybePersistTerminalFromBodyFireAndForget(opts.upstreamId, response, waitUntil); - return response; - }); - - const result = await streamingProviderCall( - upstreamFetch, - (body, parserOptions) => parseAnthropicMessagesStream(body, { ...parserOptions, onSseFrame: observeRawSseFrame }), - upstreamModelId, - opts.signal, - ); - - if (!result.ok && result.response.status === 401 && !accessToken.freshlyMinted && !alreadyRetried) { - // Cached token rejected; invalidate so the next mint reads stale=null, - // then re-enter with a fresh-minted token. A second 401 (alreadyRetried - // == true) means the refresh_token itself is the problem and the - // operator has to re-import — surface the 401 verbatim so the gateway - // boundary reports the real upstream message rather than masking it. - await invalidateClaudeCodeAccessToken({ - upstreamId: opts.upstreamId, - repo: getProviderRepo().upstreams, - }); - const ensured = await ensureOrSession503(opts, upstreamModelId); - if ('modelKey' in ensured) return ensured; - return await performUpstreamCall(opts, upstreamModelId, ensured, true); - } - - if (!result.ok) return result; - return { - ...result, - events: observedClaudeCodeMessagesStream(result.events, { - upstreamId: opts.upstreamId, - model: upstreamModelId, - headers: responseHeaders ?? result.headers ?? new Headers(), - signal: opts.signal, - frames: rawSseFrames, - rawFrameCount: () => rawSseFrameCount, - }), - }; -}; diff --git a/packages/provider-claude-code/src/headers.ts b/packages/provider-claude-code/src/headers.ts index 782ebd555a..06498143a4 100644 --- a/packages/provider-claude-code/src/headers.ts +++ b/packages/provider-claude-code/src/headers.ts @@ -29,7 +29,7 @@ const STAINLESS_PACKAGE_VERSION = '0.94.0'; // // `X-Stainless-Helper-Method: stream` is always set: every /v1/messages // call we make to Anthropic is a streaming call (see `wireBody.stream = -// true` in fetch.ts), so this matches real CC's wire output captured in +// true` in backend.ts), so this matches real CC's wire output captured in // `@anthropic-ai/claude-code@2.1.10` cli.js and sub2api's pinned set // (gateway_service.go:7427-7429 / allowedHeaders list at :432). const STAINLESS_BASE = { diff --git a/packages/provider-claude-code/src/index.ts b/packages/provider-claude-code/src/index.ts index ec630466d5..fee1f4fa67 100644 --- a/packages/provider-claude-code/src/index.ts +++ b/packages/provider-claude-code/src/index.ts @@ -19,7 +19,7 @@ export * from './detection.ts'; export * from './headers.ts'; export * from './log.ts'; export * from './quota.ts'; -export * from './interceptors/anthropic-messages/system-blocks.ts'; +export * from './system-blocks.ts'; export * from './pricing.ts'; -export * from './fetch.ts'; +export * from './backend.ts'; export * from './provider.ts'; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts deleted file mode 100644 index 61d2df5943..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS } from '@floway-dev/protocols/anthropic-messages'; - -// Real Claude Code always sends `max_tokens` and `temperature` on every -// request body. Anthropic's `/v1/messages` requires `max_tokens` and 422s -// without it; `temperature` is technically optional upstream, but its -// absence is a CC-shape fingerprint failure that the plan-billing detector -// keys on. Third-party callers (cline, aider, custom integrations) routinely -// omit one or both, expecting the gateway to backfill. -// -// Sub2api (`backend/internal/service/gateway_service.go:1301-1314`, -// rev 4a5665da5b2c6b83c4597844ea6e573746c821b1) unconditionally backfills -// both: `max_tokens` to 128000 and `temperature` to 1. We mirror the same -// unconditional fill but cap `max_tokens` to the model's advertised output -// limit when present (`limits.max_output_tokens`), falling back to the -// gateway-wide ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS (8192) — sub2api's hardcoded -// 128000 ignores per-model output caps which we don't want to reproduce. -// -// Positioned at the head of the chain so the rest of the re-mimicry steps -// see a fully-formed payload. Caller-supplied values are never overwritten. -export const backfillRequiredFieldsPayload =

>(payload: P, model: Pick): P => { - const next = { ...payload }; - - next.max_tokens ??= model.limits.max_output_tokens ?? ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS; - next.temperature ??= 1; - - return next; -}; - -export const backfillRequiredFields = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = backfillRequiredFieldsPayload(ctx.payload, ctx.model); - return await run(); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts deleted file mode 100644 index 21d5e58c70..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts +++ /dev/null @@ -1,67 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { AnthropicMessagesMessage, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; - -// Synthetic assistant turn that closes the hoisted user/assistant pair so the -// upstream's role-alternation guard stays satisfied. The text is visible to -// the upstream and may echo back in completion text, so keep it minimal. -// -// Two independent OAuth-mimicry impls (sub2api `gateway_service.go:4486` -// and claude-relay-service) converged on this exact literal; divergence is -// a likely detector signal, so we match. -const SYNTHETIC_ACK = 'Understood. I will follow these instructions.'; - -// On the re-mimicry path the upstream's `system` slot is reserved for the -// three CC-mimicry blocks (billing / identity / default template). Any -// caller-supplied system content must therefore move OUT of `system` before -// those three blocks are injected, otherwise the operator's instructions -// would be lost. The convention sub2api and Parrot both ship — and the one -// real CC clients fall back to when their own system text exceeds the bare -// identity block — is to fold the original system text into the head of -// `messages` as a synthetic user/assistant turn. The acknowledgement keeps -// the conversation alternation valid so the upstream's role-alternation -// guard doesn't fire. -// -// Non-text fields on blocks (citations, cache_control) are intentionally -// dropped — the wrapped turn is best-effort recovery of the operator's -// intent. -// -// References: -// - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/gateway_service.go#L4480-L4486 -export const hoistUserSystemToMessagesPayload =

>(payload: P): P => { - const system: string | AnthropicMessagesTextBlock[] | undefined = payload.system; - let captured = ''; - if (typeof system === 'string') { - captured = system; - } else if (system !== undefined) { - captured = system - .map(block => block.text) - .filter(text => typeof text === 'string' && text.length > 0) - .join('\n\n'); - } - // inject-billing-block et al rebuild `system` from scratch as a three-block - // array; removing the field here keeps the boundary mutation self-contained. - const nextPayload = { ...payload }; - delete nextPayload.system; - - if (captured !== '') { - // Wrapper format `[System Instructions]\n${text}` matches sub2api - // `gateway_service.go:4480` byte-for-byte; the synthetic user content is - // emitted as a structured `[{type:"text",text:...}]` block (the shape - // both reference impls and real CC use) rather than a raw string. - const synthetic: AnthropicMessagesMessage[] = [ - { role: 'user', content: [{ type: 'text', text: `[System Instructions]\n${captured}` }] }, - { role: 'assistant', content: [{ type: 'text', text: SYNTHETIC_ACK }] }, - ]; - nextPayload.messages = [...synthetic, ...nextPayload.messages]; - } - - return nextPayload; -}; - -export const hoistUserSystemToMessages = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = hoistUserSystemToMessagesPayload(ctx.payload); - return await run(); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts deleted file mode 100644 index fef4d69900..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts +++ /dev/null @@ -1,55 +0,0 @@ -// Claude Code re-mimicry chain. Runs only on the re-mimicry path (callAnthropicMessages -// short-circuits to passthrough when the inbound request is already CC-shaped). -// -// Order matters: -// 1. backfill-required-fields fills max_tokens and temperature when -// the third-party caller omitted them. -// Runs first so the rest of the chain -// (and downstream fingerprint compute -// in inject-billing-block) sees the -// fully-formed CC wire shape. -// 2. synthesize-metadata-user-id runs early so session_id derives from -// the operator's real first user -// message rather than the synthetic -// ... pair that hoist -// injects when system text is present. -// Two conversations sharing a system -// prompt must NOT share a session id, -// or prompt-cache routing and -// rate-limit accounting collapse. -// 3. hoist-user-system-to-messages captures the caller's system text into -// a synthetic user/assistant pair so the -// three mimicry blocks below own -// `payload.system`. -// 4. inject-billing-block system[0]: per-request cc_version / -// cch=00000 fingerprint. -// 5. inject-identity-block system[1]: canonical CC identity text. -// 6. inject-default-template system[2]: cached boilerplate template -// (carries cache_control:ephemeral). -// -// The on-wire `model` field is set in `fetch.ts` from -// `opts.model.providerData.upstreamModelId` so the chain never has to rewrite -// it — the catalog id is already Anthropic's public alias and the dated -// upstream id is read straight off `opts.model.providerData`. - -import { backfillRequiredFields } from './backfill-required-fields.ts'; -import { hoistUserSystemToMessages } from './hoist-user-system-to-messages.ts'; -import { injectBillingBlock } from './inject-billing-block.ts'; -import { injectDefaultTemplate } from './inject-default-template.ts'; -import { injectIdentityBlock } from './inject-identity-block.ts'; -import { synthesizeMetadataUserId } from './synthesize-metadata-user-id.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; - -export type { AnthropicMessagesBoundaryCtx } from './types.ts'; - -export const CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY: readonly Interceptor>[] = [ - backfillRequiredFields, - synthesizeMetadataUserId, - hoistUserSystemToMessages, - injectBillingBlock, - injectIdentityBlock, - injectDefaultTemplate, -]; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts deleted file mode 100644 index 87f5772e4d..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { buildBillingBlock, computeCcVersionFingerprint } from './system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import { CLAUDE_CLI_VERSION } from '../../headers.ts'; - -// Drops the per-request `cc_version=${VERSION}.${FP}` billing block at the -// head of `system`. This must run BEFORE inject-identity-block / -// inject-default-template so the order on the wire matches the byte-for-byte -// CC shape: system[0] billing, system[1] identity, system[2] default -// template (the cached one). -// -// Hoist must have run first so any caller-supplied system text is already -// captured into `messages`; this interceptor unconditionally starts a fresh -// `system` array. -// -// The fingerprint runs on the post-hoist payload deliberately. That is the -// shape Anthropic will actually see on the wire, so the fingerprint must -// reflect it — fingerprinting the pre-hoist shape would compute a different -// value than what the request body settles to and break CC mimicry. -export const injectBillingBlockPayload =

>(payload: P): P => { - const fingerprint = computeCcVersionFingerprint(CLAUDE_CLI_VERSION, payload); - const block = buildBillingBlock(CLAUDE_CLI_VERSION, fingerprint); - payload = { ...payload, system: [block] }; - return payload; -}; - -export const injectBillingBlock = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = injectBillingBlockPayload(ctx.payload); - return await run(); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts deleted file mode 100644 index 927d477998..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { DEFAULT_TEMPLATE_BLOCK } from './system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; - -// Anthropic's prompt-caching API rejects requests that carry more than four -// `cache_control` markers; the docs state "the API enforces a maximum of 4 -// cache points per request" and the budget is shared across system blocks, -// tools, and message content blocks. Reference: -// https://platform.claude.com/docs/en/build-with-claude/prompt-caching -// sub2api's `gateway_service.go:71` ports the same constant -// (`maxCacheControlBlocks = 4`) and enforces it in `enforceCacheControlLimit` -// at line 4581. -const ANTHROPIC_CACHE_BREAKPOINT_CAP = 4; - -// Counts `cache_control` markers the caller has already placed across tools, -// the `system` array (when in block form), and each message's content blocks. -// Mirrors sub2api's `collectCacheControlPaths`. The billing and identity -// blocks injected earlier in this chain carry no `cache_control`, so they -// never contribute here. -const countCacheBreakpoints = (payload: Pick): number => { - let count = 0; - - if (payload.tools) { - for (const tool of payload.tools) { - if ('cache_control' in tool && tool.cache_control) count++; - } - } - - if (Array.isArray(payload.system)) { - for (const block of payload.system) { - if (block.cache_control) count++; - } - } - - for (const message of payload.messages) { - if (typeof message.content === 'string') continue; - for (const block of message.content) { - if ('cache_control' in block && block.cache_control) count++; - } - } - - return count; -}; - -// Appends the cached default-template block at system[2]. The block normally -// carries `cache_control: { type: 'ephemeral', ttl: '5m' }` so Anthropic's -// prompt cache keys on the (billing, identity, template) triplet up to here; -// the billing block intentionally sits BEFORE this cache breakpoint so its -// per-request fingerprint never invalidates the cached prefix. -// -// When the caller's payload is already at or near the four-breakpoint cap, -// adding our breakpoint would push the request over the limit and Anthropic -// rejects with HTTP 400. In that case we still inject the template text -// (real CC's shape relies on three system blocks) but demote it to an -// un-cached block. The caller's own breakpoints stay where they are — if -// the caller already exceeds the cap, that overage is theirs to fix. -// -// Chain order in ./index.ts guarantees `system` is already a fresh array -// when this runs. -export const injectDefaultTemplatePayload =

>(payload: P): P => { - if (!Array.isArray(payload.system)) { - throw new Error('inject-default-template: expected system to be an array (inject-billing-block must run first)'); - } - const system = payload.system; - const callerBreakpoints = countCacheBreakpoints(payload); - const wouldOverflowBreakpointCap = callerBreakpoints >= ANTHROPIC_CACHE_BREAKPOINT_CAP; - const templateBlock = wouldOverflowBreakpointCap - ? { type: 'text' as const, text: DEFAULT_TEMPLATE_BLOCK.text } - : DEFAULT_TEMPLATE_BLOCK; - payload = { ...payload, system: [...system, templateBlock] }; - return payload; -}; - -export const injectDefaultTemplate = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = injectDefaultTemplatePayload(ctx.payload); - return await run(); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts deleted file mode 100644 index 1b80776ffa..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { IDENTITY_BLOCK } from './system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; - -// system[1]; relies on injectBillingBlock having materialized payload.system as an array (see ./index.ts chain order). -export const injectIdentityBlockPayload =

>(payload: P): P => { - const system = payload.system as AnthropicMessagesTextBlock[]; - payload = { ...payload, system: [...system, IDENTITY_BLOCK] }; - return payload; -}; - -export const injectIdentityBlock = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = injectIdentityBlockPayload(ctx.payload); - return await run(); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts deleted file mode 100644 index 18096f2433..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { sha256 } from '@noble/hashes/sha2.js'; -import { bytesToHex } from '@noble/hashes/utils.js'; -import { v4 } from 'uuid'; - -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { AnthropicMessagesMessage, AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; - -// Real CC includes `metadata.user_id` on every /v1/messages request: a JSON -// envelope `{device_id, account_uuid, session_id}` (v2.1.78+) or the legacy -// underscore-delimited string (pre-2.1.78). Anthropic's detector treats a -// missing user_id as one of several CC-shape failures, and even a third- -// party id seeds the same plan-billing routing key the CLI uses — so we -// always populate it on the re-mimicry path. -// -// Deterministic ids: device_id is per-upstream stable (one CC "device" per -// upstream record), session_id is per-payload (so a multi-turn conversation -// re-uses the same session_id when the prefix repeats — the property -// prompt-cache routing relies on). Stability comes from sha256 over -// (upstream id + nonce / payload prefix); randomness would defeat the -// upstream's cache and burn rate-limit slots faster. -// -// account_uuid is the empty string by convention — real CC uses the empty -// string for personal accounts and a real UUID for org members; sub2api -// observed the upstream accepts an empty string regardless. We always -// emit empty rather than leaking the operator's actual account uuid into -// per-request mimicry. -// -// References: -// - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/metadata_userid.go#L15 - -export const synthesizeMetadataUserIdPayload =

>(payload: P, upstreamId: string): P => { - const existing = payload.metadata?.user_id; - if (typeof existing === 'string' && existing.length > 0) return payload; - - const deviceId = deviceIdForUpstream(upstreamId); - const sessionId = sessionIdForPayload(upstreamId, payload); - const userId = JSON.stringify({ device_id: deviceId, account_uuid: '', session_id: sessionId }); - - payload = { ...payload, metadata: { ...payload.metadata, user_id: userId } }; - return payload; -}; - -export const synthesizeMetadataUserId = async ( - ctx: AnthropicMessagesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = synthesizeMetadataUserIdPayload(ctx.payload, ctx.upstreamId); - return await run(); -}; - -// 64-hex (32-byte) device_id, matching the format real CC emits. See -// https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/gateway_prompt_test.go#L17 -const deviceIdForUpstream = (upstreamId: string): string => - sha256Hex(`claude-code-device:${upstreamId}`); - -// Session id derives from the upstream id plus the first user message text, -// so multi-turn conversations of the same conversation prefix re-use the -// same session id (good for prompt cache) but different conversations get -// different ids. Mirrors the strategy the Codex provider uses for its own -// session-id derivation, with a per-upstream salt so two upstreams running -// the same script don't collide. -const sessionIdForPayload = (upstreamId: string, payload: Pick): string => { - const firstUser = firstUserMessageText(payload.messages); - return sha256Uuidv4(`claude-code-session:${upstreamId}${firstUser}`); -}; - -const firstUserMessageText = (messages: AnthropicMessagesMessage[]): string => { - for (const msg of messages) { - if (msg.role !== 'user') continue; - if (typeof msg.content === 'string') return msg.content; - return msg.content - .map(part => part.type === 'text' ? part.text : '') - .filter(Boolean) - .join('\n'); - } - return ''; -}; - -const sha256Hex = (input: string): string => bytesToHex(sha256(new TextEncoder().encode(input))); - -const sha256Uuidv4 = (input: string): string => { - return v4({ random: sha256(new TextEncoder().encode(input)) }); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/types.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/types.ts deleted file mode 100644 index ddd1ed502c..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/types.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderModel } from '@floway-dev/provider'; - -// Boundary ctx for Claude Code Anthropic Messages interceptors. The chain runs only on -// the re-mimicry path; callAnthropicMessages decides shaped-vs-unshaped before -// entering the chain. `upstreamId` is required by synthesize-metadata-user-id -// to derive deterministic device/session ids that stay stable per upstream -// across requests (so prompt-cache hits depend on conversation content only, -// not on per-call randomness). -export interface AnthropicMessagesBoundaryCtx { - payload: AnthropicMessagesPayload; - readonly model: ProviderModel; - readonly upstreamId: string; -} diff --git a/packages/provider-claude-code/src/models.ts b/packages/provider-claude-code/src/models.ts index c931c3b067..71dcf23454 100644 --- a/packages/provider-claude-code/src/models.ts +++ b/packages/provider-claude-code/src/models.ts @@ -10,7 +10,7 @@ // 4.6+ and Fable 5+ return with the alias already (no date), // so the alias derivation is the identity. The catalog id we publish is // always the alias; the original /v1/models id rides on -// `providerData.upstreamModelId` so the wire fetch in `fetch.ts` and the +// `providerData.upstreamModelId` so the wire fetch in `backend.ts` and the // pricing table key by the per-revision id. import { CLAUDE_CODE_HEADERS_SONNET_OPUS } from './headers.ts'; diff --git a/packages/provider-claude-code/src/provider.ts b/packages/provider-claude-code/src/provider.ts index 62eeaa13e2..f4685796d8 100644 --- a/packages/provider-claude-code/src/provider.ts +++ b/packages/provider-claude-code/src/provider.ts @@ -1,21 +1,14 @@ import { ensureClaudeCodeAccessToken } from './access-token.ts'; import { assertClaudeCodeUpstreamRecord } from './config.ts'; import { CLAUDE_CODE_DEFAULT_FLAGS } from './defaults.ts'; -import { isClaudeCodeShapedRequest } from './detection.ts'; -import { callClaudeCodeAnthropicMessages } from './fetch.ts'; -import { CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY, type AnthropicMessagesBoundaryCtx } from './interceptors/anthropic-messages/index.ts'; import { buildClaudeCodeCatalog, fetchClaudeCodeModelsList } from './models.ts'; import { createClaudeCodePipelines } from './pipelines.ts'; import { assertClaudeCodeUpstreamState } from './state.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { getProviderRepo, - headersForAnthropicMessagesCall, resolveEffectiveFlags, type ProviderInstance, type Provider, - type ProviderStreamResult, type UpstreamRecord, } from '@floway-dev/provider'; @@ -42,7 +35,6 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { const enabledFlags = resolveEffectiveFlags([CLAUDE_CODE_DEFAULT_FLAGS, record.flagOverrides]); const instance: ProviderInstance = { - callAlphaSearch: rejectUnsupported('callAlphaSearch'), // Catalog refresh mints an access token and hits /v1/models on every // dispatcher poll. `ensureClaudeCodeAccessToken` flips the row to // `refresh_failed` and throws `ClaudeCodeOAuthSessionTerminatedError` @@ -57,65 +49,6 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { const apiModels = await fetchClaudeCodeModelsList(access.entry.token, fetcher); return buildClaudeCodeCatalog(apiModels, enabledFlags); }, - - callAnthropicMessages: async (model, body, signal: AbortSignal | undefined, opts) => { - const ctx: AnthropicMessagesBoundaryCtx = { - payload: { ...body, model: model.id }, - model, - upstreamId: record.id, - }; - - // Detection runs on the unmodified payload plus the Claude Code - // fingerprint admitted by the provider module and Anthropic Messages boundaries. - // The re-mimicry chain would clobber operator-supplied `system` content - // and overwrite the wire shape — exactly what a CC-shaped passthrough - // needs to preserve. So the chain only runs on the unshaped path; the - // shaped path skips straight to the terminal call, which preserves the - // caller's own system blocks, metadata and tool shape rather than - // re-deriving them. The call preserves that filtered fingerprint, - // supplies the provider-owned OAuth auth, and restamps the resolved model - // id. - const looksShaped = isClaudeCodeShapedRequest({ - headers: new Headers(headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta).map(([name, value]) => [name, value])), - body: ctx.payload, - }); - - const terminal = async (): Promise> => { - // Drop `model` from the payload: callClaudeCodeAnthropicMessages re-attaches the - // dated upstream id (from `opts.model.providerData.upstreamModelId`) on - // the wire so Anthropic sees a stable per-revision id rather than the - // public alias the catalog exposes to clients. - const { model: _ignored, ...wireBody } = ctx.payload; - return await callClaudeCodeAnthropicMessages({ - upstreamId: record.id, - model, - body: wireBody, - shaped: looksShaped, - signal, - call: opts, - }); - }; - - if (looksShaped) return await terminal(); - - return await runInterceptors>( - ctx, - CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY, - terminal, - ); - }, - - // Only /v1/messages is supported; reject any other endpoint loudly so a - // dispatcher routing bug surfaces instead of a silent shape mismatch. - callAnthropicMessagesCountTokens: rejectUnsupported('callAnthropicMessagesCountTokens'), - callOpenAICompletions: rejectUnsupported('callOpenAICompletions'), - callOpenAIChatCompletions: rejectUnsupported('callOpenAIChatCompletions'), - callOpenAIResponses: rejectUnsupported('callOpenAIResponses'), - callOpenAIEmbeddings: rejectUnsupported('callOpenAIEmbeddings'), - callOpenAIImagesGenerations: rejectUnsupported('callOpenAIImagesGenerations'), - callOpenAIImagesEdits: rejectUnsupported('callOpenAIImagesEdits'), - callOpenAIAudioTranscriptions: rejectUnsupported('callOpenAIAudioTranscriptions'), - callRerank: rejectUnsupported('callRerank'), }; return { @@ -130,6 +63,3 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { instance, }; }; - -const rejectUnsupported = (capability: string) => (): Promise => - Promise.reject(new Error(`Claude Code provider does not implement ${capability}`)); diff --git a/packages/provider-claude-code/src/stages/authenticate.ts b/packages/provider-claude-code/src/stages/authenticate.ts index 3c6c75c63c..ec50782d61 100644 --- a/packages/provider-claude-code/src/stages/authenticate.ts +++ b/packages/provider-claude-code/src/stages/authenticate.ts @@ -1,6 +1,6 @@ import { ensureClaudeCodeAccessToken } from '../access-token.ts'; import { ClaudeCodeOAuthSessionTerminatedError } from '../auth/oauth.ts'; -import { isRateLimitedNow, synthetic429, synthetic503 } from '../fetch.ts'; +import { isRateLimitedNow, synthetic429, synthetic503 } from '../backend.ts'; import type { ClaudeCodeAuthenticatedRequest, ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; import { readClaudeCodeUpstreamState } from '../state.ts'; import { takeHttpResponse } from '@floway-dev/http/pipeline'; diff --git a/packages/provider-claude-code/src/stages/backfill-required-fields.ts b/packages/provider-claude-code/src/stages/backfill-required-fields.ts index 3845261bae..3a047932af 100644 --- a/packages/provider-claude-code/src/stages/backfill-required-fields.ts +++ b/packages/provider-claude-code/src/stages/backfill-required-fields.ts @@ -1,7 +1,34 @@ -import { backfillRequiredFieldsPayload } from '../interceptors/anthropic-messages/backfill-required-fields.ts'; import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import type { ProviderChatResponse } from '@floway-dev/provider'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderModel, ProviderChatResponse } from '@floway-dev/provider'; + +// Real Claude Code always sends `max_tokens` and `temperature` on every +// request body. Anthropic's `/v1/messages` requires `max_tokens` and 422s +// without it; `temperature` is technically optional upstream, but its +// absence is a CC-shape fingerprint failure that the plan-billing detector +// keys on. Third-party callers (cline, aider, custom integrations) routinely +// omit one or both, expecting the gateway to backfill. +// +// Sub2api (`backend/internal/service/gateway_service.go:1301-1314`, +// rev 4a5665da5b2c6b83c4597844ea6e573746c821b1) unconditionally backfills +// both: `max_tokens` to 128000 and `temperature` to 1. We mirror the same +// unconditional fill but cap `max_tokens` to the model's advertised output +// limit when present (`limits.max_output_tokens`), falling back to the +// gateway-wide ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS (8192) — sub2api's hardcoded +// 128000 ignores per-model output caps which we don't want to reproduce. +// +// Positioned at the head of the chain so the rest of the re-mimicry steps +// see a fully-formed payload. Caller-supplied values are never overwritten. +export const backfillRequiredFieldsPayload =

>(payload: P, model: Pick): P => { + const next = { ...payload }; + + next.max_tokens ??= model.limits.max_output_tokens ?? ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS; + next.temperature ??= 1; + + return next; +}; export const backfillRequiredFields = defineStage, ProviderChatResponse<'anthropicMessages'>>({ name: 'backfillRequiredFields', diff --git a/packages/provider-claude-code/src/stages/decode-response.ts b/packages/provider-claude-code/src/stages/decode-response.ts index 37f7a85475..fca2a1df6b 100644 --- a/packages/provider-claude-code/src/stages/decode-response.ts +++ b/packages/provider-claude-code/src/stages/decode-response.ts @@ -1,4 +1,4 @@ -import { observedClaudeCodeMessagesStream, STREAM_DIAGNOSTIC_FRAME_DATA_CHARS, STREAM_DIAGNOSTIC_FRAME_LIMIT, type StreamDiagnosticFrame } from '../fetch.ts'; +import { observedClaudeCodeMessagesStream, STREAM_DIAGNOSTIC_FRAME_DATA_CHARS, STREAM_DIAGNOSTIC_FRAME_LIMIT, type StreamDiagnosticFrame } from '../backend.ts'; import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, setRelease } from '@floway-dev/pipeline'; diff --git a/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts b/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts index 7d9a709ef5..05ae158868 100644 --- a/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts +++ b/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts @@ -1,8 +1,65 @@ -import { hoistUserSystemToMessagesPayload } from '../interceptors/anthropic-messages/hoist-user-system-to-messages.ts'; import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesMessage, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderChatResponse } from '@floway-dev/provider'; +// Synthetic assistant turn that closes the hoisted user/assistant pair so the +// upstream's role-alternation guard stays satisfied. The text is visible to +// the upstream and may echo back in completion text, so keep it minimal. +// +// Two independent OAuth-mimicry impls (sub2api `gateway_service.go:4486` +// and claude-relay-service) converged on this exact literal; divergence is +// a likely detector signal, so we match. +const SYNTHETIC_ACK = 'Understood. I will follow these instructions.'; + +// On the re-mimicry path the upstream's `system` slot is reserved for the +// three CC-mimicry blocks (billing / identity / default template). Any +// caller-supplied system content must therefore move OUT of `system` before +// those three blocks are injected, otherwise the operator's instructions +// would be lost. The convention sub2api and Parrot both ship — and the one +// real CC clients fall back to when their own system text exceeds the bare +// identity block — is to fold the original system text into the head of +// `messages` as a synthetic user/assistant turn. The acknowledgement keeps +// the conversation alternation valid so the upstream's role-alternation +// guard doesn't fire. +// +// Non-text fields on blocks (citations, cache_control) are intentionally +// dropped — the wrapped turn is best-effort recovery of the operator's +// intent. +// +// References: +// - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/gateway_service.go#L4480-L4486 +export const hoistUserSystemToMessagesPayload =

>(payload: P): P => { + const system: string | AnthropicMessagesTextBlock[] | undefined = payload.system; + let captured = ''; + if (typeof system === 'string') { + captured = system; + } else if (system !== undefined) { + captured = system + .map(block => block.text) + .filter(text => typeof text === 'string' && text.length > 0) + .join('\n\n'); + } + // inject-billing-block et al rebuild `system` from scratch as a three-block + // array; removing the field here keeps the payload transformation self-contained. + const nextPayload = { ...payload }; + delete nextPayload.system; + + if (captured !== '') { + // Wrapper format `[System Instructions]\n${text}` matches sub2api + // `gateway_service.go:4480` byte-for-byte; the synthetic user content is + // emitted as a structured `[{type:"text",text:...}]` block (the shape + // both reference impls and real CC use) rather than a raw string. + const synthetic: AnthropicMessagesMessage[] = [ + { role: 'user', content: [{ type: 'text', text: `[System Instructions]\n${captured}` }] }, + { role: 'assistant', content: [{ type: 'text', text: SYNTHETIC_ACK }] }, + ]; + nextPayload.messages = [...synthetic, ...nextPayload.messages]; + } + + return nextPayload; +}; + export const hoistUserSystemToMessages = defineStage, ProviderChatResponse<'anthropicMessages'>>({ name: 'hoistUserSystemToMessages', through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, diff --git a/packages/provider-claude-code/src/stages/inject-billing-block.ts b/packages/provider-claude-code/src/stages/inject-billing-block.ts index 2f2a1e6ce9..3b9d32d4b6 100644 --- a/packages/provider-claude-code/src/stages/inject-billing-block.ts +++ b/packages/provider-claude-code/src/stages/inject-billing-block.ts @@ -1,8 +1,31 @@ -import { injectBillingBlockPayload } from '../interceptors/anthropic-messages/inject-billing-block.ts'; +import { CLAUDE_CLI_VERSION } from '../headers.ts'; import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { buildBillingBlock, computeCcVersionFingerprint } from '../system-blocks.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderChatResponse } from '@floway-dev/provider'; +// Drops the per-request `cc_version=${VERSION}.${FP}` billing block at the +// head of `system`. This must run BEFORE inject-identity-block / +// inject-default-template so the order on the wire matches the byte-for-byte +// CC shape: system[0] billing, system[1] identity, system[2] default +// template (the cached one). +// +// Hoist must have run first so any caller-supplied system text is already +// captured into `messages`; this stage unconditionally starts a fresh +// `system` array. +// +// The fingerprint runs on the post-hoist payload deliberately. That is the +// shape Anthropic will actually see on the wire, so the fingerprint must +// reflect it — fingerprinting the pre-hoist shape would compute a different +// value than what the request body settles to and break CC mimicry. +export const injectBillingBlockPayload =

>(payload: P): P => { + const fingerprint = computeCcVersionFingerprint(CLAUDE_CLI_VERSION, payload); + const block = buildBillingBlock(CLAUDE_CLI_VERSION, fingerprint); + payload = { ...payload, system: [block] }; + return payload; +}; + export const injectBillingBlock = defineStage, ProviderChatResponse<'anthropicMessages'>>({ name: 'injectBillingBlock', through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, diff --git a/packages/provider-claude-code/src/stages/inject-default-template.ts b/packages/provider-claude-code/src/stages/inject-default-template.ts index 88a42a88c2..e6583d1089 100644 --- a/packages/provider-claude-code/src/stages/inject-default-template.ts +++ b/packages/provider-claude-code/src/stages/inject-default-template.ts @@ -1,8 +1,78 @@ -import { injectDefaultTemplatePayload } from '../interceptors/anthropic-messages/inject-default-template.ts'; import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { DEFAULT_TEMPLATE_BLOCK } from '../system-blocks.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderChatResponse } from '@floway-dev/provider'; +// Anthropic's prompt-caching API rejects requests that carry more than four +// `cache_control` markers; the docs state "the API enforces a maximum of 4 +// cache points per request" and the budget is shared across system blocks, +// tools, and message content blocks. Reference: +// https://platform.claude.com/docs/en/build-with-claude/prompt-caching +// sub2api's `gateway_service.go:71` ports the same constant +// (`maxCacheControlBlocks = 4`) and enforces it in `enforceCacheControlLimit` +// at line 4581. +const ANTHROPIC_CACHE_BREAKPOINT_CAP = 4; + +// Counts `cache_control` markers the caller has already placed across tools, +// the `system` array (when in block form), and each message's content blocks. +// Mirrors sub2api's `collectCacheControlPaths`. The billing and identity +// blocks injected earlier in this chain carry no `cache_control`, so they +// never contribute here. +const countCacheBreakpoints = (payload: Pick): number => { + let count = 0; + + if (payload.tools) { + for (const tool of payload.tools) { + if ('cache_control' in tool && tool.cache_control) count++; + } + } + + if (Array.isArray(payload.system)) { + for (const block of payload.system) { + if (block.cache_control) count++; + } + } + + for (const message of payload.messages) { + if (typeof message.content === 'string') continue; + for (const block of message.content) { + if ('cache_control' in block && block.cache_control) count++; + } + } + + return count; +}; + +// Appends the cached default-template block at system[2]. The block normally +// carries `cache_control: { type: 'ephemeral', ttl: '5m' }` so Anthropic's +// prompt cache keys on the (billing, identity, template) triplet up to here; +// the billing block intentionally sits BEFORE this cache breakpoint so its +// per-request fingerprint never invalidates the cached prefix. +// +// When the caller's payload is already at or near the four-breakpoint cap, +// adding our breakpoint would push the request over the limit and Anthropic +// rejects with HTTP 400. In that case we still inject the template text +// (real CC's shape relies on three system blocks) but demote it to an +// un-cached block. The caller's own breakpoints stay where they are — if +// the caller already exceeds the cap, that overage is theirs to fix. +// +// Chain order in ../pipelines.ts guarantees `system` is already a fresh array +// when this runs. +export const injectDefaultTemplatePayload =

>(payload: P): P => { + if (!Array.isArray(payload.system)) { + throw new Error('inject-default-template: expected system to be an array (inject-billing-block must run first)'); + } + const system = payload.system; + const callerBreakpoints = countCacheBreakpoints(payload); + const wouldOverflowBreakpointCap = callerBreakpoints >= ANTHROPIC_CACHE_BREAKPOINT_CAP; + const templateBlock = wouldOverflowBreakpointCap + ? { type: 'text' as const, text: DEFAULT_TEMPLATE_BLOCK.text } + : DEFAULT_TEMPLATE_BLOCK; + payload = { ...payload, system: [...system, templateBlock] }; + return payload; +}; + export const injectDefaultTemplate = defineStage, ProviderChatResponse<'anthropicMessages'>>({ name: 'injectDefaultTemplate', through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, diff --git a/packages/provider-claude-code/src/stages/inject-identity-block.ts b/packages/provider-claude-code/src/stages/inject-identity-block.ts index 45e3194f46..391b49fd17 100644 --- a/packages/provider-claude-code/src/stages/inject-identity-block.ts +++ b/packages/provider-claude-code/src/stages/inject-identity-block.ts @@ -1,8 +1,16 @@ -import { injectIdentityBlockPayload } from '../interceptors/anthropic-messages/inject-identity-block.ts'; import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { IDENTITY_BLOCK } from '../system-blocks.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderChatResponse } from '@floway-dev/provider'; +// system[1]; relies on injectBillingBlock having materialized payload.system as an array (see ../pipelines.ts chain order). +export const injectIdentityBlockPayload =

>(payload: P): P => { + const system = payload.system as AnthropicMessagesTextBlock[]; + payload = { ...payload, system: [...system, IDENTITY_BLOCK] }; + return payload; +}; + export const injectIdentityBlock = defineStage, ProviderChatResponse<'anthropicMessages'>>({ name: 'injectIdentityBlock', through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, diff --git a/packages/provider-claude-code/src/stages/observe-response.ts b/packages/provider-claude-code/src/stages/observe-response.ts index 17c0130e2f..4644820df7 100644 --- a/packages/provider-claude-code/src/stages/observe-response.ts +++ b/packages/provider-claude-code/src/stages/observe-response.ts @@ -1,4 +1,4 @@ -import { detectTerminalSentinel, persistQuotaSnapshot, persistTerminalAccountState } from '../fetch.ts'; +import { detectTerminalSentinel, persistQuotaSnapshot, persistTerminalAccountState } from '../backend.ts'; import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; import { parseClaudeCodeQuotaHeaders } from '../quota.ts'; import { exchangeResponse, takeHttpResponse } from '@floway-dev/http/pipeline'; diff --git a/packages/provider-claude-code/src/stages/prepare-request.ts b/packages/provider-claude-code/src/stages/prepare-request.ts index 96f048dbd8..b736a09a4d 100644 --- a/packages/provider-claude-code/src/stages/prepare-request.ts +++ b/packages/provider-claude-code/src/stages/prepare-request.ts @@ -1,4 +1,4 @@ -import { ANTHROPIC_MESSAGES_ENDPOINT } from '../fetch.ts'; +import { ANTHROPIC_MESSAGES_ENDPOINT } from '../backend.ts'; import { pickClaudeCodeHeaders } from '../headers.ts'; import type { ClaudeCodeAuthenticatedRequest, ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; import { defineStage, isSecret, move, secret } from '@floway-dev/pipeline'; diff --git a/packages/provider-claude-code/src/stages/retry-access.ts b/packages/provider-claude-code/src/stages/retry-access.ts index 78994bc982..b6766315ac 100644 --- a/packages/provider-claude-code/src/stages/retry-access.ts +++ b/packages/provider-claude-code/src/stages/retry-access.ts @@ -1,6 +1,6 @@ import { ensureClaudeCodeAccessToken, invalidateClaudeCodeAccessToken } from '../access-token.ts'; import { ClaudeCodeOAuthSessionTerminatedError } from '../auth/oauth.ts'; -import { synthetic503 } from '../fetch.ts'; +import { synthetic503 } from '../backend.ts'; import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; import { takeHttpResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, secret } from '@floway-dev/pipeline'; diff --git a/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts b/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts index 2aeadcc943..1d32792125 100644 --- a/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts +++ b/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts @@ -1,10 +1,83 @@ -import { synthesizeMetadataUserIdPayload } from '../interceptors/anthropic-messages/synthesize-metadata-user-id.ts'; +import { sha256 } from '@noble/hashes/sha2.js'; +import { bytesToHex } from '@noble/hashes/utils.js'; +import { v4 } from 'uuid'; + import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesMessage, AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { ProviderChatResponse } from '@floway-dev/provider'; +// Real CC includes `metadata.user_id` on every /v1/messages request: a JSON +// envelope `{device_id, account_uuid, session_id}` (v2.1.78+) or the legacy +// underscore-delimited string (pre-2.1.78). Anthropic's detector treats a +// missing user_id as one of several CC-shape failures, and even a third- +// party id seeds the same plan-billing routing key the CLI uses — so we +// always populate it on the re-mimicry path. +// +// Deterministic ids: device_id is per-upstream stable (one CC "device" per +// upstream record), session_id is per-payload (so a multi-turn conversation +// re-uses the same session_id when the prefix repeats — the property +// prompt-cache routing relies on). Stability comes from sha256 over +// (upstream id + nonce / payload prefix); randomness would defeat the +// upstream's cache and burn rate-limit slots faster. +// +// account_uuid is the empty string by convention — real CC uses the empty +// string for personal accounts and a real UUID for org members; sub2api +// observed the upstream accepts an empty string regardless. We always +// emit empty rather than leaking the operator's actual account uuid into +// per-request mimicry. +// +// References: +// - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/metadata_userid.go#L15 + +export const synthesizeMetadataUserIdPayload =

>(payload: P, upstreamId: string): P => { + const existing = payload.metadata?.user_id; + if (typeof existing === 'string' && existing.length > 0) return payload; + + const deviceId = deviceIdForUpstream(upstreamId); + const sessionId = sessionIdForPayload(upstreamId, payload); + const userId = JSON.stringify({ device_id: deviceId, account_uuid: '', session_id: sessionId }); + + payload = { ...payload, metadata: { ...payload.metadata, user_id: userId } }; + return payload; +}; + export const synthesizeMetadataUserId = (upstreamId: string) => defineStage, ProviderChatResponse<'anthropicMessages'>>({ name: 'synthesizeMetadataUserId', through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : synthesizeMetadataUserIdPayload(facts['request.provider.payload'], upstreamId) })) }), }); + +// 64-hex (32-byte) device_id, matching the format real CC emits. See +// https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/gateway_prompt_test.go#L17 +const deviceIdForUpstream = (upstreamId: string): string => + sha256Hex(`claude-code-device:${upstreamId}`); + +// Session id derives from the upstream id plus the first user message text, +// so multi-turn conversations of the same conversation prefix re-use the +// same session id (good for prompt cache) but different conversations get +// different ids. Mirrors the strategy the Codex provider uses for its own +// session-id derivation, with a per-upstream salt so two upstreams running +// the same script don't collide. +const sessionIdForPayload = (upstreamId: string, payload: Pick): string => { + const firstUser = firstUserMessageText(payload.messages); + return sha256Uuidv4(`claude-code-session:${upstreamId}${firstUser}`); +}; + +const firstUserMessageText = (messages: AnthropicMessagesMessage[]): string => { + for (const msg of messages) { + if (msg.role !== 'user') continue; + if (typeof msg.content === 'string') return msg.content; + return msg.content + .map(part => part.type === 'text' ? part.text : '') + .filter(Boolean) + .join('\n'); + } + return ''; +}; + +const sha256Hex = (input: string): string => bytesToHex(sha256(new TextEncoder().encode(input))); + +const sha256Uuidv4 = (input: string): string => { + return v4({ random: sha256(new TextEncoder().encode(input)) }); +}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts b/packages/provider-claude-code/src/system-blocks.ts similarity index 100% rename from packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts rename to packages/provider-claude-code/src/system-blocks.ts diff --git a/packages/provider-codex/__tests__/fetch_test.ts b/packages/provider-codex/__tests__/fetch_test.ts index 20303e0ef6..cdce55b656 100644 --- a/packages/provider-codex/__tests__/fetch_test.ts +++ b/packages/provider-codex/__tests__/fetch_test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { createUpstreamStateRepoStub } from './upstream-state-repo.ts'; import { CODEX_CLI_VERSION, CODEX_ORIGINATOR, CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY, CODEX_RESPONSES_LITE_HEADER, CODEX_USER_AGENT } from '../src/constants.ts'; -import { callCodexAlphaSearch, callCodexOpenAIImagesGenerations, callCodexOpenAIResponses, callCodexOpenAIResponsesCompact, type CodexCallEffects } from '../src/fetch.ts'; +import { callCodexAlphaSearch, callCodexOpenAIImagesGenerations, callCodexOpenAIResponses, callCodexOpenAIResponsesCompact, type CodexCallEffects } from './test-utils/call.ts'; import * as responsesLite from '../src/responses-lite.ts'; import type { CodexAccessTokenEntry, CodexAccountCredential, CodexQuotaSnapshotEntryMap, CodexUpstreamState } from '../src/state.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; @@ -62,8 +62,6 @@ const seedAccountState = (overrides: Partial): void => { const readQuotaEntry = (): CodexQuotaSnapshotEntryMap | null => (currentRecord.state as CodexUpstreamState).accounts[0].quotaSnapshot; -// putCodexQuota fires-and-forgets via .catch(() => {}); yield to the task -// queue so the saveState promise resolves before the caller asserts on state. const flushMicrotasks = (): Promise => new Promise(resolve => setTimeout(resolve, 0)); beforeEach(() => { @@ -81,7 +79,7 @@ afterEach(() => vi.restoreAllMocks()); const sseResponse = (status = 200): Response => new Response( new ReadableStream({ start(c) { - c.enqueue(new TextEncoder().encode('event: response.created\ndata: {"type":"response.created"}\n\n')); + c.enqueue(new TextEncoder().encode('event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_test","object":"response","status":"completed","model":"gpt-5.4","output":[]}}\n\n')); c.close(); }, }), @@ -147,7 +145,7 @@ describe('callCodexOpenAIResponses — gates', () => { }, }, }); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const result = await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -177,7 +175,7 @@ describe('callCodexOpenAIResponses — token freshness', () => { test('reuses fresh state-cached access token without refreshing', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -191,7 +189,7 @@ describe('callCodexOpenAIResponses — token freshness', () => { refresh_token: null, accessToken: { token: 'at_only', expiresAt: null, refreshedAt: 'now' }, }); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const result = await callCodexOpenAIResponses({ upstreamId, account: accessOnlyAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -559,7 +557,7 @@ describe('Codex private Responses wire selection', () => { bodies.push(await new Response(init?.body).text()); headers.push(new Headers(init?.headers)); if (bodies.length === 1) { - selectedModel.providerData.useResponsesLite = false; + expect(Object.isFrozen(selectedModel.providerData)).toBe(true); return errorJson(401, { error: { code: 'expired_token', message: 'expired' } }); } return action === 'generate' ? sseEventsResponse([]) : compactJsonResponse(); @@ -598,7 +596,7 @@ describe('Codex private Responses wire selection', () => { headers: new Headers({ [CODEX_RESPONSES_LITE_HEADER]: 'false' }), effects: makeEffects(), call: noopUpstreamCallOptions(), }); if (result.ok) throw new Error('expected upstream error'); - expect(result.response).toBe(upstream); + expect(result.response.status).toBe(upstream.status); expect(result.response.headers.get(CODEX_RESPONSES_LITE_HEADER)).toBe(marker ?? null); expect(result.response.headers.get('content-type')).toBe('application/problem+json'); expect(result.response.statusText).toBe('Upstream failure'); @@ -611,7 +609,7 @@ describe('Codex private Responses wire selection', () => { describe('callCodexOpenAIResponses — upstream classification', () => { test('happy path: 200 → ok:true, quota persisted', async () => { seedFreshAccessToken(); - vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const result = await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -625,7 +623,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('upstream body has store:false and stream:true forced even if caller passes otherwise', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: false as unknown as true, store: true } as unknown as Parameters[0]['body'], @@ -639,7 +637,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('builds Codex responses headers and metadata from a clean set', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -719,7 +717,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { refresh_token: null, accessToken: { token: 'at_only', expiresAt: null, refreshedAt: 'now' }, }); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const result = await callCodexOpenAIResponses({ upstreamId, account, @@ -843,7 +841,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('preserves a hyphenated Codex session id for prompt cache', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -860,7 +858,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('canonicalizes downstream session_id to the Codex session-id header', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -877,7 +875,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('prefers downstream session-id over session_id when both are provided', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -894,7 +892,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('generates a Codex session id when the downstream request has none', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -908,7 +906,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('derives the same session id across turns of a stateless conversation', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const turn = { upstreamId, account: activeAccount, model, body: { @@ -932,7 +930,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('derives distinct session ids when only the instructions differ', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const call = (instructions: string) => callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { @@ -954,7 +952,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('derives distinct session ids when only the first user message differs', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const call = (content: string) => callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { @@ -976,7 +974,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('uses account.openaiDeviceId as the installation id', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const deviceId = '22222222-3333-4444-9555-666666666666'; await callCodexOpenAIResponses({ upstreamId, account: { ...activeAccount, openaiDeviceId: deviceId }, @@ -992,7 +990,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('prefers a caller-supplied installation id from client_metadata over the account device id', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: { ...activeAccount, openaiDeviceId: 'account-device-id' }, model, @@ -1014,7 +1012,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('passes through caller thread-id and x-client-request-id when distinct from session-id', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, @@ -1038,7 +1036,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('merges caller-supplied x-codex-turn-metadata extras over the synthesized blob', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, @@ -1072,7 +1070,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('reads the turn-metadata blob from the body before the header', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { @@ -1117,7 +1115,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('keeps the unbounded tool inventory in the body blob and out of the header blob', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const toolNamespacesInfo = { namespaces: [{ name: 'shell', tools: ['exec'] }] }; await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -1147,7 +1145,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('preserves caller client_metadata extras while keeping identity-mirror keys gateway-owned', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { @@ -1271,22 +1269,18 @@ describe('callCodexOpenAIResponses — upstream classification', () => { }); }); -describe('callCodexOpenAIResponses — background-write registration', () => { - // Background state writes (quota snapshot on 2xx/429, access-token put on - // 401-retry) must reach the runtime's waitUntil slot so workerd does not - // cancel them the instant the streaming response returns to the client. - // Without this, freshly-minted Codex tokens and quota snapshots get dropped - // on the floor and the next request re-mints / re-races the upstream. - test('2xx persists quota snapshot via opts.call.waitUntil', async () => { +describe('Codex Responses deferred state writes', () => { + test('2xx quota persistence completes through the run deferred lifecycle', async () => { seedFreshAccessToken(); - vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const waitUntil = vi.fn<(promise: Promise) => void>(); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: { ...noopUpstreamCallOptions(), waitUntil }, }); - expect(waitUntil).toHaveBeenCalledTimes(1); + expect(waitUntil).not.toHaveBeenCalled(); + expect(readQuotaEntry()).not.toBeNull(); }); test('401-retry persists the fresh access token before returning', async () => { @@ -1301,9 +1295,10 @@ describe('callCodexOpenAIResponses — background-write registration', () => { model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: { ...noopUpstreamCallOptions(), waitUntil }, }); - // The access-token write is awaited because its CAS result carries the - // effective plan; only the successful retry's quota write is backgrounded. - expect(waitUntil).toHaveBeenCalledTimes(1); + // The refresh CAS supplies the retry plan; drain also completes the + // successful endpoint's quota observation. + expect(waitUntil).not.toHaveBeenCalled(); + expect(readQuotaEntry()).not.toBeNull(); expect((currentRecord.state as CodexUpstreamState).accounts[0].accessToken?.token).toBe('at2'); }); }); @@ -1459,14 +1454,6 @@ describe('callCodexOpenAIImagesGenerations', () => { }); }); -// `callCodexOpenAIResponsesCompact` shares OAuth + quota + 401-retry plumbing with -// `callCodexOpenAIResponses` (both go through `prepareCodexCall` → -// `dispatchCodexHttpCall` → `refreshAccessTokenForRetry`). The streaming -// suite above pins those shared paths; this block exercises only the -// compact-specific wire contract — endpoint URL, `Accept: application/json`, -// body shape (no `stream`, no `store`), unary JSON decoding — plus the 401 -// retry on the unary endpoint to confirm the retry decision is taken from -// the bare response status (no SSE wrap in the path). const compactJsonResponse = (overrides?: Partial): Response => new Response(JSON.stringify({ id: 'resp_x', @@ -1513,7 +1500,7 @@ describe('callCodexOpenAIResponsesCompact', () => { expect(result.result.output[0]).toMatchObject({ id: 'cmp_x', type: 'compaction', encrypted_content: 'FULL_BLOB' }); }); - test('2xx persists quota snapshot via opts.call.waitUntil', async () => { + test('2xx quota persistence completes through the run deferred lifecycle', async () => { seedFreshAccessToken(); vi.spyOn(globalThis, 'fetch').mockResolvedValue(compactJsonResponse()); const waitUntil = vi.fn<(promise: Promise) => void>(); @@ -1522,7 +1509,8 @@ describe('callCodexOpenAIResponsesCompact', () => { body: { input: [] }, headers: new Headers(), effects: makeEffects(), call: { ...noopUpstreamCallOptions(), waitUntil }, }); - expect(waitUntil).toHaveBeenCalledTimes(1); + expect(waitUntil).not.toHaveBeenCalled(); + expect(readQuotaEntry()).not.toBeNull(); }); test('401 other → refresh + retry once on the compact endpoint, succeed', async () => { diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index a5e8df44a9..5a11ac4074 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -6,7 +6,7 @@ import { createCodexProvider } from '../src/provider.ts'; import type { CodexAccessTokenEntry, CodexUpstreamState } from '../src/state.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; import { move, run, setRelease, type Event } from '@floway-dev/pipeline'; -import { directFetcher, initProviderRepo, providerModelFacts, type UpstreamRecord } from '@floway-dev/provider'; +import { directFetcher, initProviderRepo, providerModelFacts, type ProviderOperationPayloads, type UpstreamRecord } from '@floway-dev/provider'; import { callProviderPipeline, collectChatProviderPipeline, noopUpstreamCallOptions, readJsonRequest, stubProviderModel } from '@floway-dev/test-utils'; const farFutureMs = Date.now() + 24 * 60 * 60 * 1000; @@ -358,7 +358,7 @@ describe('createCodexProvider', () => { test('callOpenAIResponses preserves developer messages on the Codex wire', async () => { const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); const instance = createCodexProvider(baseRecord); - const result = await instance.instance.callOpenAIResponses( + const result = await collectChatProviderPipeline(instance, 'openaiResponses', stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} } }), { input: [ @@ -368,12 +368,10 @@ describe('createCodexProvider', () => { ], stream: true, }, - 'generate', undefined, - noopUpstreamCallOptions(), - ); - expect(result.ok).toBe(true); - expect(result.action).toBe('generate'); + noopUpstreamCallOptions()); + expect(result.output).not.toBeNull(); + expect(result.facts['response.provider.responsesAction']).toBe('generate'); const init = fetchSpy.mock.calls[0]?.[1] as RequestInit | undefined; if (init === undefined) throw new Error('expected a Codex upstream request'); const body = await readJsonRequest(init) as Record; @@ -385,7 +383,7 @@ describe('createCodexProvider', () => { ]); }); - test.each(['generate', 'compact'] as const)('%s retains the full Standard body through interceptors before private Lite encoding', async action => { + test.each(['generate', 'compact'] as const)('%s retains the full Standard body through stages before private Lite encoding', async action => { const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => action === 'generate' ? sseResponse() : new Response(JSON.stringify({ id: 'cmp_1', object: 'response.compaction', output: [] }))); @@ -399,14 +397,14 @@ describe('createCodexProvider', () => { ]; const options = noopUpstreamCallOptions(); options.headers.set(CODEX_RESPONSES_LITE_HEADER, 'true'); - const result = await provider.instance.callOpenAIResponses(model, { + const result = await collectChatProviderPipeline(provider, action === 'generate' ? 'openaiResponses' : 'openaiResponsesCompact', model, { input, tools: [{ type: 'custom', name: 'patch' }], text: { verbosity: 'low' }, client_metadata: { [CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY]: 'true' }, - } as Parameters[1], action, undefined, options); - expect(result.ok).toBe(true); - expect(result.action).toBe(action); + } as ProviderOperationPayloads['openaiResponses'], undefined, options); + expect(result.output).not.toBeNull(); + expect(result.facts['response.provider.responsesAction']).toBe(action); const wire = await readJsonRequest(fetchSpy.mock.calls[0]![1] as RequestInit) as Record; expect(wire).not.toHaveProperty('instructions'); expect(wire).not.toHaveProperty('tools'); @@ -430,15 +428,13 @@ describe('createCodexProvider', () => { test('callOpenAIResponses re-reads state per request (operator re-import takes effect)', async () => { repo.getById.mockResolvedValueOnce({ ...baseRecord, state: { accounts: [{ chatgptAccountId: 'acc', refresh_token: 'rt_v1', state: 'session_terminated', state_updated_at: '2026-01-02T00:00:00Z', openaiDeviceId: '11111111-2222-4333-8444-555555555555', accessToken: null, quotaSnapshot: null }] } as CodexUpstreamState }); const instance = createCodexProvider(baseRecord); - const result = await instance.instance.callOpenAIResponses( + const result = await collectChatProviderPipeline(instance, 'openaiResponses', stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} } }), { input: [], stream: true }, - 'generate', undefined, - noopUpstreamCallOptions(), - ); - expect(result.ok).toBe(false); - if (!result.ok) expect(result.response.status).toBe(503); + noopUpstreamCallOptions()); + expect(result.output).toBeNull(); + expect(result.response!.status).toBe(503); }); test('callOpenAIImagesGenerations posts gpt-image-2 through the ChatGPT Codex endpoint', async () => { @@ -516,21 +512,7 @@ describe('createCodexProvider', () => { }); }); - test.each([ - 'callOpenAIEmbeddings', - 'callOpenAIAudioTranscriptions', - 'callOpenAIChatCompletions', - 'callAnthropicMessagesCountTokens', - 'callAnthropicMessages', - ] as const)('%s returns a synthetic 405 (data plane never dispatches these to Codex)', async method => { - const instance = createCodexProvider(baseRecord); - const model = stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} } }); - // @ts-expect-error: each method has a different body type; we only assert - // the synthetic 405 envelope is what comes back. - const result = await instance.instance[method](model, undefined, noopUpstreamCallOptions()) as { response: Response }; - expect(result.response.status).toBe(405); - const body = await result.response.json() as { error: { type: string; message: string } }; - expect(body.error.type).toBe('method_not_allowed'); - expect(body.error.message).toMatch(/codex/i); + test.each(['openaiEmbeddings', 'openaiAudioTranscriptions', 'openaiChatCompletions', 'anthropicMessagesCountTokens', 'anthropicMessages'] as const)('%s has no Codex dispatch pipeline', operation => { + expect(createCodexProvider(baseRecord).pipelines[operation]).toBeUndefined(); }); }); diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts b/packages/provider-codex/__tests__/stages/action-pivot_test.ts similarity index 67% rename from packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts rename to packages/provider-codex/__tests__/stages/action-pivot_test.ts index 37cc55dc39..95f75db047 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts +++ b/packages/provider-codex/__tests__/stages/action-pivot_test.ts @@ -1,42 +1,30 @@ import { afterEach, beforeEach, expect, test, vi } from 'vitest'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; -import { createUpstreamStateRepoStub } from '../../upstream-state-repo.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import { initProviderRepo, type ProviderOpenAIResponsesResult, type UpstreamRecord } from '@floway-dev/provider'; +import { createUpstreamStateRepoStub } from '../upstream-state-repo.ts'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import { initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; import { assertEquals, readJsonRequest } from '@floway-dev/test-utils'; -// Codex's terminal handler in provider.ts switches on `ctx.action` (the -// post-chain value), so an interceptor that flips it mid-chain reroutes -// dispatch. Drive the contract by swapping CODEX_OPENAI_RESPONSES_BOUNDARY for one -// that ends in a pivot interceptor (generate → compact), then call with -// action='generate' and a generate-shaped body. The wire request seen at -// the upstream MUST hit /codex/responses/compact AND the body MUST NOT -// carry generate-only fields (tools/reasoning/temperature/...) — the -// per-action narrowing through `toCompactPayloadShape` is what closes that -// gap. -const pivotGenerateToCompact: Interceptor = async (ctx, run) => { - ctx.action = 'compact'; - ctx.payload = { - ...ctx.payload, - instructions: 'Late instructions', - tools: [...ctx.payload.tools ?? [], { type: 'custom', name: 'late_tool' }], - }; - return await run(); -}; - -vi.mock('../../../src/interceptors/openai-responses/index.ts', async () => { - const original = await vi.importActual('../../../src/interceptors/openai-responses/index.ts'); - return { - ...original, - CODEX_OPENAI_RESPONSES_BOUNDARY: [...original.CODEX_OPENAI_RESPONSES_BOUNDARY, pivotGenerateToCompact], - }; +// A provider stage can pivot an initially selected generate operation before private +// wire preparation. Both Standard and Lite compaction projection must see its new content. +vi.mock('../../src/stages/inject-default-instructions.ts', async () => { + const original = await vi.importActual('../../src/stages/inject-default-instructions.ts'); + const { defineStage, move } = await import('@floway-dev/pipeline'); + const pivot = defineStage, Record, Record, Record>({ + name: 'pivotCodexResponsesToCompact', + through: { request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload', 'request.provider.responsesAction'], provides: ['request.provider.payload', 'request.provider.responsesAction'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload'] as CanonicalOpenAIResponsesPayload; + return move({ ...await next(move({ ...facts, 'request.provider.responsesAction': 'compact', 'request.provider.payload': { ...payload, instructions: 'Late instructions', tools: [...payload.tools ?? [], { type: 'custom', name: 'late_tool' }] } })) }); + }, + }); + return { ...original, injectCodexDefaultInstructions: pivot }; }); // Imports below MUST follow the vi.mock so the provider module resolves // against the mocked chain on first import. -const { createCodexProvider } = await import('../../../src/provider.ts'); -const { noopUpstreamCallOptions, stubProviderModel } = await import('@floway-dev/test-utils'); +const { createCodexProvider } = await import('../../src/provider.ts'); +const { collectChatProviderPipeline, noopUpstreamCallOptions, stubProviderModel } = await import('@floway-dev/test-utils'); const farFutureMs = Date.now() + 24 * 60 * 60 * 1000; @@ -93,7 +81,7 @@ test.each([true, false])('Codex projects a post-chain compact pivot after catalo }); const instance = createCodexProvider(baseRecord); - const result = await instance.instance.callOpenAIResponses( + const result = await collectChatProviderPipeline(instance, 'openaiResponses', stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} }, providerData: { useResponsesLite } }), { input: [{ type: 'message', role: 'user', content: 'hi' }], @@ -104,13 +92,11 @@ test.each([true, false])('Codex projects a post-chain compact pivot after catalo stream: true, parallel_tool_calls: false, }, - 'generate', undefined, - noopUpstreamCallOptions(), - ); + noopUpstreamCallOptions()); - if (!result.ok) throw new Error('expected ok result'); - if (result.action !== 'compact') throw new Error(`expected compact variant after pivot, got ${result.action}`); + expect(result.output).toMatchObject({ kind: 'value' }); + expect(result.facts['response.provider.responsesAction']).toBe('compact'); if (compactUrl === undefined) throw new Error('expected /codex/responses/compact to be hit'); if (compactBody === undefined) throw new Error('expected compact body capture'); diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts b/packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts similarity index 80% rename from packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts rename to packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts index 03df7358f5..70527099c4 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts +++ b/packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts @@ -1,15 +1,14 @@ import { test } from 'vitest'; -import { injectDefaultInstructions } from '../../../src/interceptors/openai-responses/inject-default-instructions.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { injectCodexDefaultInstructions } from '../../src/stages/inject-default-instructions.ts'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test', headers: new Headers() }); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -19,7 +18,7 @@ const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBo test('injects the default when instructions is absent', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); - await injectDefaultInstructions(ctx, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -27,7 +26,7 @@ test('injects the default when instructions is absent', async () => { test('injects the default when instructions is an empty string', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: '' }); - await injectDefaultInstructions(ctx, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -35,7 +34,7 @@ test('injects the default when instructions is an empty string', async () => { test('injects the default when instructions is null', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: null }); - await injectDefaultInstructions(ctx, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -43,7 +42,7 @@ test('injects the default when instructions is null', async () => { test('preserves a caller-supplied instructions string', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: 'You are a pirate.' }); - await injectDefaultInstructions(ctx, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, 'You are a pirate.'); }); @@ -62,7 +61,7 @@ test.each([ instructions: value as unknown as string, }); - await injectDefaultInstructions(ctx, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, value); }, @@ -78,7 +77,7 @@ test('injects the default and preserves input items it does not own', async () = ], }); - await injectDefaultInstructions(ctx, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); assertEquals(ctx.payload.input, [ diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts b/packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts similarity index 85% rename from packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts rename to packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts index 43d2c72fb7..88b4923f30 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts +++ b/packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts @@ -1,15 +1,14 @@ import { test } from 'vitest'; -import { stripUnsupportedFields } from '../../../src/interceptors/openai-responses/strip-unsupported-fields.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { stripCodexUnsupportedFields } from '../../src/stages/strip-unsupported-response-fields.ts'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, assertFalse, stubProviderModel } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, assertEquals, assertFalse, stubProviderModel } from '@floway-dev/test-utils'; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test', headers: new Headers() }); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -19,7 +18,7 @@ const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBo test('drops every field Codex rejects with Unsupported parameter', async () => { // The fields enumerated below are the full set Codex's ChatGPT-subscription // path rejects; keeping the assertion exhaustive guards against silent - // drift if the constant inside the interceptor is edited without updating + // drift if the constant inside the stage is edited without updating // its rationale. Several entries (frequency_penalty, presence_penalty, // user, stream_options) are not on the canonical payload and reach Codex only // through a permissive caller. `prompt_cache_retention` is modeled but @@ -40,7 +39,7 @@ test('drops every field Codex rejects with Unsupported parameter', async () => { stream_options: { include_usage: true }, } as unknown as CanonicalOpenAIResponsesPayload); - await stripUnsupportedFields(ctx, okEvents); + await applyProviderStage(stripCodexUnsupportedFields, ctx, okEvents); assertFalse('max_output_tokens' in ctx.payload); assertFalse('temperature' in ctx.payload); @@ -65,7 +64,7 @@ test('leaves supported fields intact', async () => { temperature: 0.7, }); - await stripUnsupportedFields(ctx, okEvents); + await applyProviderStage(stripCodexUnsupportedFields, ctx, okEvents); assertEquals(ctx.payload.model, 'gpt-test'); assertEquals(ctx.payload.input, [{ type: 'message', role: 'user', content: 'hello' }]); @@ -80,7 +79,7 @@ test('payload without any unsupported fields is preserved as-is', async () => { const payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }; const ctx = invocation(payload); - await stripUnsupportedFields(ctx, okEvents); + await applyProviderStage(stripCodexUnsupportedFields, ctx, okEvents); assertEquals(ctx.payload, { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); }); diff --git a/packages/provider-codex/__tests__/test-utils/call.ts b/packages/provider-codex/__tests__/test-utils/call.ts new file mode 100644 index 0000000000..ddf08439c0 --- /dev/null +++ b/packages/provider-codex/__tests__/test-utils/call.ts @@ -0,0 +1,36 @@ +import type { CodexBackendCallBase, CodexCallEffects } from '../../src/backend.ts'; +import { createCodexPipelines } from '../../src/pipelines.ts'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { Provider, ProviderModel, ProviderStreamResult, ProviderOperationPayloads, UpstreamCallOptions } from '@floway-dev/provider'; +import { callProviderPipeline, collectChatProviderPipeline } from '@floway-dev/test-utils'; + +interface CallOptions extends CodexBackendCallBase { model: ProviderModel } +interface ResponsesOptions extends CallOptions { body: Omit } +const providerFor = (opts: CallOptions, fallbackPlanType: string | undefined = undefined): Provider => ({ + upstreamId: opts.upstreamId, kind: 'codex', name: 'Codex test', inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: { getProvidedModels: async () => [] }, + pipelines: createCodexPipelines({ upstreamId: opts.upstreamId, readAccount: async () => opts.account, effects: opts.effects, fallbackPlanType }), +}); +const optionsFor = (opts: CallOptions): UpstreamCallOptions => ({ ...opts.call, headers: opts.headers }); +const replay = async function* (frames: readonly ProtocolFrame[]) { yield* frames; }; +export const callCodexOpenAIResponses = async (opts: ResponsesOptions): Promise> => { + const result = await collectChatProviderPipeline(providerFor(opts), 'openaiResponses', opts.model, opts.body, opts.signal, optionsFor(opts)); + if (result.output !== null && 'kind' in result.output && result.output.kind === 'stream') { + const exchange = result.facts['response.http.exchange']; + if (exchange.type === 'transportFailure') throw exchange.error; + return { ok: true, modelKey: result.facts['response.provider.modelKey'], events: replay(result.frames), headers: new Headers(exchange.headers.map(([name, value]): [string, string] => [name, value])) }; + } + if (result.response === null) throw new Error('Expected Responses stream or HTTP failure'); + return { ok: false, modelKey: result.facts['response.provider.modelKey'], response: result.response }; +}; +export const callCodexOpenAIResponsesCompact = async (opts: ResponsesOptions): Promise<{ ok: true; result: OpenAIResponsesCompactionResult; modelKey: string } | { ok: false; response: Response; modelKey: string }> => { + const result = await collectChatProviderPipeline(providerFor(opts), 'openaiResponsesCompact', opts.model, opts.body, opts.signal, optionsFor(opts)); + if (result.output !== null && 'kind' in result.output && result.output.kind === 'value') return { ok: true, modelKey: result.facts['response.provider.modelKey'], result: result.output.body }; + if (result.response === null) throw new Error('Expected compaction value or HTTP failure'); + return { ok: false, modelKey: result.facts['response.provider.modelKey'], response: result.response }; +}; +export const callCodexAlphaSearch = (opts: CallOptions & { body: Record }) => callProviderPipeline(providerFor(opts), 'alphaSearch', opts.model, opts.body, opts.signal, optionsFor(opts)); +export const callCodexOpenAIImagesGenerations = (opts: CallOptions & { body: Omit; fallbackPlanType: string | undefined }) => callProviderPipeline(providerFor(opts, opts.fallbackPlanType), 'openaiImagesGenerations', opts.model, opts.body as ProviderOperationPayloads['openaiImagesGenerations'], opts.signal, optionsFor(opts)); +export type { CodexCallEffects }; diff --git a/packages/provider-codex/package.json b/packages/provider-codex/package.json index 3e70f76ebf..c75f8b4b12 100644 --- a/packages/provider-codex/package.json +++ b/packages/provider-codex/package.json @@ -12,7 +12,6 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", "uuid": "^14.0.1" diff --git a/packages/provider-codex/src/fetch.ts b/packages/provider-codex/src/backend.ts similarity index 50% rename from packages/provider-codex/src/fetch.ts rename to packages/provider-codex/src/backend.ts index dc4a0cbc02..c7c01bae57 100644 --- a/packages/provider-codex/src/fetch.ts +++ b/packages/provider-codex/src/backend.ts @@ -1,21 +1,9 @@ import { CodexAccessOnlyCredentialError, codexPlanObservation, ensureCodexAccessToken, invalidateCodexAccessToken, mintCodexAccessToken, type CodexPlanObservation } from './access-token.ts'; import { isObject } from './auth/guards.ts'; import { CodexOAuthSessionTerminatedError } from './auth/oauth.ts'; -import { - CODEX_BACKEND_BASE, - CODEX_CLI_VERSION, - CODEX_ALPHA_SEARCH_PATH, - CODEX_OPENAI_IMAGES_EDITS_PATH, - CODEX_OPENAI_IMAGES_GENERATIONS_PATH, - CODEX_ORIGINATOR, - CODEX_OPENAI_RESPONSES_COMPACT_PATH, - CODEX_OPENAI_RESPONSES_PATH, - CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY, - CODEX_RESPONSES_LITE_HEADER, - CODEX_USER_AGENT, -} from './constants.ts'; +import { CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY } from './constants.ts'; import { sha256JsonUuid, uuidV7 } from './ids.ts'; -import { codexModelUsesResponsesLite, codexPlanSupportsImages } from './models.ts'; +import { codexModelUsesResponsesLite } from './models.ts'; import { hasCodexQuotaReading, parseCodexQuotaHeaders, @@ -23,22 +11,13 @@ import { } from './quota.ts'; import { encodeCodexResponsesLiteRequest, - restoreCodexResponsesCompactionResult, - restoreCodexResponsesFrames, type CodexResponsesBody, type CodexResponsesLiteRequest, } from './responses-lite.ts'; -import { restoreCodexResponsesOutput } from './responses-output.ts'; import type { CodexAccessTokenEntry, CodexAccountCredential } from './state.ts'; -import { isEventStreamMediaType } from '@floway-dev/protocols/common'; -import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesInputItem, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { parseOpenAIResponsesStream, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; -import { jsonRequestBody, serializeOpenAIImagesEditsJsonPayload, type OpenAIImagesEditsRequest, type ProviderCallResult, type ProviderModel, type ProviderStreamResult, type ReplayableBody, streamingProviderCall, type UpstreamCallOptions } from '@floway-dev/provider'; - -export type ProviderCompactionResult = - | { ok: true; result: OpenAIResponsesCompactionResult; modelKey: string } - | { ok: false; response: Response; modelKey: string }; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; +import { toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; +import type { ProviderModel, UpstreamCallOptions } from '@floway-dev/provider'; // Hooks for repo-side state transitions. Refresh-token rotations and // terminal-state transitions go through the repo; access-token and quota @@ -62,86 +41,12 @@ export interface CodexBackendCallBase { call: UpstreamCallOptions; } -export interface CallCodexOpenAIResponsesOptions extends CodexBackendCallBase { +interface CodexResponsesContentInput extends Pick { body: Omit; } -export interface CallCodexOpenAIResponsesCompactOptions extends CodexBackendCallBase { - body: Omit; -} - -export interface CallCodexAlphaSearchOptions extends CodexBackendCallBase { - body: Record; -} - -export interface CallCodexOpenAIImagesGenerationsOptions extends CodexBackendCallBase { - body: Omit; - // Null when the account identity carries no plan claim; the image gate - // treats an unknown plan as allowed (fail open). - fallbackPlanType: string | undefined; -} - -export interface CallCodexOpenAIImagesEditsOptions extends CodexBackendCallBase { - request: OpenAIImagesEditsRequest; - fallbackPlanType: string | undefined; -} - type CodexOpenAIResponsesBody = CodexResponsesBody; -export const callCodexOpenAIResponses = async (opts: CallCodexOpenAIResponsesOptions): Promise> => { - const prepared = prepareCodexResponsesRequest(opts, 'generate'); - const ready = await prepareCodexCall(opts); - if (!ready.ok) return { ok: false, modelKey: opts.model.id, response: ready.response }; - const result = await performStreamingOpenAIResponsesCall(opts, prepared, ready.accessToken, false); - if (!result.ok || !result.headers?.has(CODEX_RESPONSES_LITE_HEADER)) return result; - const headers = new Headers(result.headers); - headers.delete(CODEX_RESPONSES_LITE_HEADER); - return { ...result, headers }; -}; - -export const callCodexOpenAIResponsesCompact = async (opts: CallCodexOpenAIResponsesCompactOptions): Promise => { - const prepared = prepareCodexResponsesRequest(opts, 'compact'); - const ready = await prepareCodexCall(opts); - if (!ready.ok) return { ok: false, modelKey: opts.model.id, response: ready.response }; - return await performUnaryCompactCall(opts, prepared, ready.accessToken, false); -}; - -export const callCodexAlphaSearch = async (opts: CallCodexAlphaSearchOptions): Promise => { - const requestId = stringField(opts.body, 'id') ?? uuidV7(); - const normalized = { ...opts, body: { ...opts.body, id: requestId } }; - const ready = await prepareCodexCall(normalized); - if (!ready.ok) return { modelKey: normalized.model.id, response: ready.response }; - return await performAlphaSearchCall(normalized, ready.accessToken, false); -}; - -const prepareCodexImageCall = async (opts: CodexBackendCallBase & { fallbackPlanType: string | undefined }): Promise<{ ok: true; accessToken: CodexAccessTokenEntry; effectivePlan: CodexPlanObservation; turnId: string } | { ok: false; response: Response }> => { - const ready = await prepareCodexCall(opts); - if (!ready.ok) return { ok: false, response: ready.response }; - // An account whose plan is unknown falls back to a fail-open sentinel: the - // upstream gate only withholds images from an explicitly `free` plan. - const effectivePlan = codexPlanObservation(ready.accessToken) - ?? (opts.fallbackPlanType === undefined ? { planType: '' } : { planType: opts.fallbackPlanType }); - if (!codexPlanSupportsImages(effectivePlan.planType)) return { ok: false, response: imageUnavailableResult(opts.model.id).response }; - const turnId = trimHeader(opts.headers, 'x-codex-image-turn-id') ?? uuidV7(); - return { ok: true, accessToken: ready.accessToken, effectivePlan, turnId }; -}; - -export const callCodexOpenAIImagesGenerations = async (opts: CallCodexOpenAIImagesGenerationsOptions): Promise => { - const prepared = await prepareCodexImageCall(opts); - if (!prepared.ok) return { modelKey: opts.model.id, response: prepared.response }; - const body = { ...opts.body, model: opts.model.id }; - const request: CodexImageCallRequest = { path: CODEX_OPENAI_IMAGES_GENERATIONS_PATH, body, turnId: prepared.turnId }; - return await performImageCall(opts, request, prepared.accessToken, prepared.effectivePlan, false); -}; - -export const callCodexOpenAIImagesEdits = async (opts: CallCodexOpenAIImagesEditsOptions): Promise => { - const prepared = await prepareCodexImageCall(opts); - if (!prepared.ok) return { modelKey: opts.model.id, response: prepared.response }; - const body = await serializeOpenAIImagesEditsJsonPayload(opts.request, opts.model.id); - const request: CodexImageCallRequest = { path: CODEX_OPENAI_IMAGES_EDITS_PATH, body, turnId: prepared.turnId }; - return await performImageCall(opts, request, prepared.accessToken, prepared.effectivePlan, false); -}; - export const prepareCodexCall = async (opts: CodexBackendCallBase): Promise<{ ok: true; accessToken: CodexAccessTokenEntry } | { ok: false; response: Response }> => { if (opts.account.state !== 'active') { return { ok: false, response: synthetic503(`Codex upstream is ${opts.account.state}`) }; @@ -241,7 +146,7 @@ const parseClientTurnMetadataJson = (raw: string | null): Record): Record | null => +const callerTurnMetadata = (opts: Pick, clientMetadata: Record): Record | null => parseClientTurnMetadataJson(stringField(clientMetadata, 'x-codex-turn-metadata')) ?? parseClientTurnMetadataJson(trimHeader(opts.headers, 'x-codex-turn-metadata')); @@ -259,7 +164,7 @@ const IDENTITY_MIRRORED_CLIENT_METADATA_KEYS = new Set([ ]); const buildCodexRequestIdentity = ( - opts: CodexBackendCallBase, + opts: Pick, body: CodexOpenAIResponsesBody, clientMetadata: Record, clientTurnMetadata: Record | null, @@ -392,15 +297,15 @@ const buildCodexClientMetadata = (identity: CodexRequestIdentity, turnMetadataJs 'x-codex-turn-metadata': turnMetadataJson, }); -interface PreparedCodexResponsesRequest { +export interface PreparedCodexResponsesRequest { identity: CodexRequestIdentity; turnMetadataJson: CodexTurnMetadataJson; lite?: CodexResponsesLiteRequest; - body: ReplayableBody; + body: Record; } export const prepareCodexResponsesContent = ( - opts: CallCodexOpenAIResponsesOptions | CallCodexOpenAIResponsesCompactOptions, + opts: CodexResponsesContentInput, action: 'generate' | 'compact', ): Omit & { body: Record } => { const clientMetadata = { ...clientCodexClientMetadata(opts.body) }; @@ -428,14 +333,6 @@ export const prepareCodexResponsesContent = ( }; }; -const prepareCodexResponsesRequest = ( - opts: CallCodexOpenAIResponsesOptions | CallCodexOpenAIResponsesCompactOptions, - action: 'generate' | 'compact', -): PreparedCodexResponsesRequest => { - const prepared = prepareCodexResponsesContent(opts, action); - return { ...prepared, body: jsonRequestBody(prepared.body) }; -}; - const buildCodexOpenAIResponsesBody = ( wire: CodexResponsesBody, modelKey: string, @@ -478,106 +375,6 @@ const buildCodexOpenAIResponsesCompactBody = ( return body; }; -interface CodexHttpCallRequest { - accessToken: string; - path: string; - accept: string; - body: ReplayableBody; - identity: CodexRequestIdentity; - turnMetadataJson: string | null; - responsesLite: boolean; -} - -// One upstream round-trip with quota-header persistence and terminal-401 -// classification. The returned Response is what the caller relays: -// - 2xx: caller decodes the body (SSE for /responses, JSON for /responses/compact) -// - 429: quota is already snapshotted; return verbatim -// - 401: an access-only credential preserves the upstream response verbatim -// and flips the row terminal, because it has nothing to retry with; on a -// renewable credential `token_invalidated` maps to a synthetic 503 and any -// other 401 is rebuilt with a re-readable body so the caller can decide to -// retry with a fresh access token -// - other: returned verbatim -const postCodexJson = async ( - opts: CodexBackendCallBase, - request: { - path: string; - accessToken: string; - body: Record; - headers: Headers; - quotaPolicy: 'always' | 'when-present'; - }, -): Promise => { - const { path, accessToken, body, headers, quotaPolicy } = request; - headers.set('authorization', `Bearer ${accessToken}`); - // A null account id omits the header rather than sending an empty one: the - // upstream reads absence as "whichever account this bearer belongs to". - if (opts.account.chatgptAccountId !== null) { - headers.set('chatgpt-account-id', opts.account.chatgptAccountId); - } - const response = await opts.call.wrapUpstreamCall(() => opts.call.fetcher(`${CODEX_BACKEND_BASE}${path}`, { - method: 'POST', - headers, - body: jsonRequestBody(body), - signal: opts.signal, - })); - return await classifyCodexHttpResponse(opts, response, quotaPolicy); -}; - -const dispatchCodexHttpCall = async ( - opts: CodexBackendCallBase, - request: CodexHttpCallRequest, -): Promise => { - const { accessToken, path, accept, body, identity, turnMetadataJson, responsesLite } = request; - const headers = new Headers(); - headers.set('authorization', `Bearer ${accessToken}`); - if (opts.account.chatgptAccountId !== null) { - headers.set('chatgpt-account-id', opts.account.chatgptAccountId); - } - headers.set('originator', CODEX_ORIGINATOR); - headers.set('user-agent', CODEX_USER_AGENT); - headers.set('version', CODEX_CLI_VERSION); - headers.set('accept', accept); - headers.set('content-type', 'application/json'); - headers.set('session-id', identity.sessionId); - headers.set('thread-id', identity.threadId); - headers.set('x-client-request-id', identity.clientRequestId); - headers.set('x-codex-window-id', identity.windowId); - if (turnMetadataJson !== null) headers.set('x-codex-turn-metadata', turnMetadataJson); - if (responsesLite) headers.set(CODEX_RESPONSES_LITE_HEADER, 'true'); - - const response = await opts.call.wrapUpstreamCall(() => opts.call.fetcher(`${CODEX_BACKEND_BASE}${path}`, { - method: 'POST', - headers, - body, - signal: opts.signal, - })); - - return await classifyCodexHttpResponse(opts, response); -}; - -export const classifyCodexHttpResponse = async ( - opts: CodexBackendCallBase, - response: Response, - quotaPolicy: 'always' | 'when-present' = 'always', -): Promise => { - if (response.ok) { - persistCodexQuotaObservation(opts, response, false, quotaPolicy); - return response; - } - - if (response.status === 429) { - persistCodexQuotaObservation(opts, response, true, quotaPolicy); - return response; - } - - if (response.status === 401) { - return await classifyCodexUnauthorizedResponse(opts, response, decodeCodexUpstreamError(await response.text())); - } - - return response; -}; - export const writeCodexQuotaObservation = ( opts: CodexBackendCallBase, response: Response, @@ -590,32 +387,6 @@ export const writeCodexQuotaObservation = ( : putCodexQuota(opts.upstreamId, opts.account.chatgptAccountId, snapshot); }; -const persistCodexQuotaObservation = ( - opts: CodexBackendCallBase, - response: Response, - isRateLimited: boolean, - policy: 'always' | 'when-present', -): void => { - const write = writeCodexQuotaObservation(opts, response, isRateLimited, policy); - if (write !== null) registerBackgroundWrite(opts, write); -}; - -const dispatchCodexImageCall = async ( - opts: CodexBackendCallBase, - request: CodexImageCallDispatchRequest, -): Promise => { - const { accessToken, path, body, turnId } = request; - const headers = new Headers({ - originator: trimHeader(opts.headers, 'originator') ?? CODEX_ORIGINATOR, - 'user-agent': CODEX_USER_AGENT, - version: CODEX_CLI_VERSION, - accept: 'application/json', - 'content-type': 'application/json', - 'x-codex-image-turn-id': turnId, - }); - return await postCodexJson(opts, { path, accessToken, body, headers, quotaPolicy: 'when-present' }); -}; - // Recover from a 401 without deleting a sibling's newer credential: invalidate // only the exact token that failed, reuse a winner already stored by another // request, otherwise force a fresh coalesced mint. The resulting CAS write is @@ -623,7 +394,7 @@ const dispatchCodexImageCall = async ( // // Every call site gates this behind `refresh_token !== null`: an access-only // credential has nothing to re-mint from, its 401 was already classified as -// terminal in `classifyCodexHttpResponse`, and the verbatim upstream response +// terminal in `observeCodexResponse`, and the verbatim upstream response // is what reaches the client. export const refreshAccessTokenForRetry = async ( opts: CodexBackendCallBase, @@ -653,37 +424,6 @@ export const refreshAccessTokenForRetry = async ( } }; -// The 401-retry gate's per-call inputs. The dispatchers bundle their access -// token, the response they saw, the recursion switch, and the retry / failure -// callbacks in one object rather than threading six positional arguments. -interface RetryCodexAccess401Options { - accessToken: CodexAccessTokenEntry; - response: Response | null; - alreadyRetried: boolean; - run: (fresh: CodexAccessTokenEntry) => Promise; - onRefreshFailure: (response: Response) => T; - fallbackPlan?: CodexPlanObservation; -} - -// The 401 retry gate every call dispatcher shares: on a renewable credential -// that has not already been retried, rotate the failed access token and re-run -// the operation once; if the refresh fails, relay the dispatcher's own failure -// result. The gate — a 401 on a credential with a refresh token, not already -// retried — is folded in here, so callers pass the response they saw and only -// branch on the retried outcome. An access-only credential's 401 is already -// classified terminal in `classifyCodexHttpResponse`, and the verbatim upstream -// response reaches the client. -const retryCodexAccess401 = async ( - opts: CodexBackendCallBase, - options: RetryCodexAccess401Options, -): Promise<{ retried: true; value: T } | { retried: false }> => { - const { accessToken, response, alreadyRetried, run, onRefreshFailure, fallbackPlan } = options; - if (response?.status !== 401 || opts.account.refresh_token === null || alreadyRetried) return { retried: false }; - const fresh = await refreshAccessTokenForRetry(opts, accessToken, fallbackPlan); - if (!fresh.ok) return { retried: true, value: onRefreshFailure(fresh.response) }; - return { retried: true, value: await run(fresh.accessToken) }; -}; - const mergeRetryPlan = ( entry: CodexAccessTokenEntry, fallback: CodexPlanObservation | undefined, @@ -696,173 +436,6 @@ const mergeRetryPlan = ( }; }; -const performStreamingOpenAIResponsesCall = async ( - opts: CallCodexOpenAIResponsesOptions, - prepared: PreparedCodexResponsesRequest, - accessToken: CodexAccessTokenEntry, - alreadyRetried: boolean, -): Promise> => { - const upstreamFetch = dispatchCodexHttpCall(opts, { - accessToken: accessToken.token, - path: CODEX_OPENAI_RESPONSES_PATH, - accept: 'text/event-stream', - body: prepared.body, - identity: prepared.identity, - turnMetadataJson: prepared.turnMetadataJson.header, - responsesLite: prepared.lite !== undefined, - }).then(ensureSseContentType); - - const lite = prepared.lite; - const result = await streamingProviderCall( - upstreamFetch, - (stream, parserOpts) => { - const frames = parseOpenAIResponsesStream(stream, parserOpts); - return restoreCodexResponsesOutput(lite === undefined - ? frames - : restoreCodexResponsesFrames(frames, lite.callableIdentities, lite.requestEchoes)); - }, - opts.model.id, - opts.signal, - ); - - const attempt = await retryCodexAccess401( - opts, - { - accessToken, - response: result.ok ? null : result.response, - alreadyRetried, - run: fresh => performStreamingOpenAIResponsesCall(opts, prepared, fresh, true), - onRefreshFailure: resp => ({ ok: false as const, modelKey: opts.model.id, response: resp }), - }, - ); - if (attempt.retried) return attempt.value; - - return result; -}; - -const performUnaryCompactCall = async ( - opts: CallCodexOpenAIResponsesCompactOptions, - prepared: PreparedCodexResponsesRequest, - accessToken: CodexAccessTokenEntry, - alreadyRetried: boolean, -): Promise => { - const response = await dispatchCodexHttpCall(opts, { - accessToken: accessToken.token, - path: CODEX_OPENAI_RESPONSES_COMPACT_PATH, - accept: 'application/json', - body: prepared.body, - identity: prepared.identity, - turnMetadataJson: prepared.turnMetadataJson.header, - responsesLite: prepared.lite !== undefined, - }); - - const attempt = await retryCodexAccess401( - opts, - { - accessToken, - response, - alreadyRetried, - run: fresh => performUnaryCompactCall(opts, prepared, fresh, true), - onRefreshFailure: resp => ({ ok: false as const, modelKey: opts.model.id, response: resp }), - }, - ); - if (attempt.retried) return attempt.value; - - if (!response.ok) return { ok: false, modelKey: opts.model.id, response }; - - const result = await response.json() as OpenAIResponsesCompactionResult; - return { - ok: true, - modelKey: opts.model.id, - result: prepared.lite === undefined - ? result - : restoreCodexResponsesCompactionResult(result, prepared.lite.callableIdentities, prepared.lite.generatedPrefix), - }; -}; - -const performAlphaSearchCall = async ( - opts: CallCodexAlphaSearchOptions, - accessToken: CodexAccessTokenEntry, - alreadyRetried: boolean, -): Promise => { - const requestId = stringField(opts.body, 'id'); - if (requestId === null) throw new Error('Normalized Codex alpha search request is missing id'); - const identity: CodexRequestIdentity = { - installationId: opts.account.openaiDeviceId, - sessionId: requestId, - threadId: requestId, - clientRequestId: requestId, - turnId: uuidV7(), - windowId: `${requestId}:0`, - }; - const turnMetadataJson = trimHeader(opts.headers, 'x-codex-turn-metadata'); - const response = await dispatchCodexHttpCall(opts, { - accessToken: accessToken.token, - path: CODEX_ALPHA_SEARCH_PATH, - accept: 'application/json', - body: jsonRequestBody({ ...opts.body, model: opts.model.id }), - identity, - turnMetadataJson, - responsesLite: false, - }); - - const attempt = await retryCodexAccess401( - opts, - { - accessToken, - response, - alreadyRetried, - run: fresh => performAlphaSearchCall(opts, fresh, true), - onRefreshFailure: resp => ({ modelKey: opts.model.id, response: resp }), - }, - ); - if (attempt.retried) return attempt.value; - return { modelKey: opts.model.id, response }; -}; - -// The fixed per-request inputs for one Codex image call. The path, body, and -// turn id are stable for the whole call, so the entry sites bundle them once -// and retries re-pass the same bundle rather than threading three positional -// arguments on every recursion. -interface CodexImageCallRequest { - path: string; - body: Record; - turnId: string; -} - -// The image-call bundle plus the resolved bearer, mirroring CodexHttpCallRequest -// for the HTTP call path. The entry sites bundle the stable path/body/turnId -// once; retries re-pass the same bundle with a fresh access token rather than -// threading four positional arguments on every recursion. -type CodexImageCallDispatchRequest = CodexImageCallRequest & { accessToken: string }; - -const performImageCall = async ( - opts: CodexBackendCallBase, - request: CodexImageCallRequest, - accessToken: CodexAccessTokenEntry, - effectivePlan: CodexPlanObservation, - alreadyRetried: boolean, -): Promise => { - const response = await dispatchCodexImageCall(opts, { accessToken: accessToken.token, ...request }); - const attempt = await retryCodexAccess401( - opts, - { - accessToken, - response, - alreadyRetried, - run: async entry => { - const refreshedPlan = codexPlanObservation(entry) ?? effectivePlan; - if (!codexPlanSupportsImages(refreshedPlan.planType)) return imageUnavailableResult(opts.model.id); - return await performImageCall(opts, request, entry, refreshedPlan, true); - }, - onRefreshFailure: resp => ({ modelKey: opts.model.id, response: resp }), - fallbackPlan: effectivePlan, - }, - ); - if (attempt.retried) return attempt.value; - return { modelKey: opts.model.id, response }; -}; - export interface CodexUpstreamError { readonly rawText: string; readonly body: unknown; @@ -904,16 +477,6 @@ export const classifyCodexUnauthorizedResponse = async (opts: CodexBackendCallBa return new Response(rawText, { status: 401, statusText: response.statusText, headers: response.headers }); }; -const imageUnavailableResult = (modelKey: string): ProviderCallResult => ({ - modelKey, - response: new Response(JSON.stringify({ - error: { - type: 'image_tools_unavailable', - message: 'ChatGPT Free accounts do not provide Codex image tools.', - }, - }), { status: 403, headers: { 'content-type': 'application/json' } }), -}); - const synthetic503 = (message: string): Response => new Response(JSON.stringify({ error: { type: 'codex_upstream_unavailable', message } }), { status: 503, headers: { 'content-type': 'application/json' }, @@ -923,23 +486,3 @@ const codexRefreshFailed = async (opts: CodexBackendCallBase, err: CodexOAuthSes await opts.effects.persistTerminalState('refresh_failed', err.upstreamMessage); return { ok: false, response: synthetic503(`Codex refresh failed: ${err.upstreamMessage}`) }; }; - -// Codex backend serves SSE without setting `content-type: text/event-stream` -// (observed in production: only x-codex-* + standard CDN headers come back). -// The shared `streamingProviderCall` rejects 2xx responses lacking the SSE -// content-type as a contract violation, so we synthesize the header on the -// way through. Body stream is preserved verbatim. -const ensureSseContentType = (response: Response): Response => { - if (!response.ok || isEventStreamMediaType(response.headers.get('content-type'))) return response; - const headers = new Headers(response.headers); - headers.set('content-type', 'text/event-stream'); - return new Response(response.body, { status: response.status, statusText: response.statusText, headers }); -}; - -// Hand best-effort writes to waitUntil so workerd does not cancel them when -// the streaming response returns; the swallow guards against recoverable -// noise (transient storage errors, a state_json write that lost every one of -// its retries) tripping the request. -const registerBackgroundWrite = (opts: CodexBackendCallBase, write: Promise): void => { - opts.call.waitUntil(write.catch(() => {})); -}; diff --git a/packages/provider-codex/src/interceptors/openai-responses/index.ts b/packages/provider-codex/src/interceptors/openai-responses/index.ts deleted file mode 100644 index 379c26f386..0000000000 --- a/packages/provider-codex/src/interceptors/openai-responses/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -// Codex-only OpenAI Responses workarounds. The chain is a boundary the Codex provider -// runs inside its own call methods, so the gateway main flow never knows that -// Codex has OpenAI Responses interceptors at all. - -import { injectDefaultInstructions } from './inject-default-instructions.ts'; -import { stripUnsupportedFields } from './strip-unsupported-fields.ts'; -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; - -// Order rationale: neither interceptor below reads or writes a field the -// other touches, so order is positional only. -// -// Codex interceptors are pure payload/header mutators, so the chain's only -// terminal — the streaming `generate` + non-streaming `compact` dispatch — -// returns its `ProviderOpenAIResponsesResult` directly without any per-frame -// lift/lower step. -export const CODEX_OPENAI_RESPONSES_BOUNDARY: readonly Interceptor[] = [ - injectDefaultInstructions, - stripUnsupportedFields, -]; diff --git a/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts b/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts deleted file mode 100644 index 0b9a6d189b..0000000000 --- a/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; - -// ChatGPT-subscription catalog models reject missing or empty `instructions`. -// Native and translated callers may omit the field, so the provider supplies a -// neutral value at its boundary. Other values remain upstream-owned validation. -// https://github.com/im4codes/imcodes/blob/5f769d933dfd679e3a4d670183b0384a1baf62cd/src/agent/providers/codex-sdk.ts#L560-L579 -export const defaultInstructionsPayload =

>(payload: P): P => { - const instructions = payload.instructions; - return instructions === undefined || instructions === null || instructions === '' - ? { ...payload, instructions: "You're a helpful assistant." } - : payload; -}; - -export const injectDefaultInstructions = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = defaultInstructionsPayload(ctx.payload); - return await run(); -}; diff --git a/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts b/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts deleted file mode 100644 index e4b2ba9c9e..0000000000 --- a/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; - -// Codex backend rejects requests carrying any of these fields with a -// `Unsupported parameter: ` 4xx. They are regular OpenAI Responses API -// fields that the ChatGPT-subscription path does not honor. Source-protocol -// translators legitimately set max_output_tokens / temperature / top_p (the -// caller's request might carry them), so we strip them at the Codex target -// boundary rather than at translation time, where they remain valid for -// other providers. -const CODEX_UNSUPPORTED_BODY_FIELDS = [ - 'max_output_tokens', - 'temperature', - 'top_p', - 'frequency_penalty', - 'presence_penalty', - 'user', - 'metadata', - 'prompt_cache_retention', - 'safety_identifier', - 'stream_options', -] as const; - -export const supportedResponsesPayload =

>(payload: P): P => - Object.fromEntries(Object.entries(payload).filter(([key]) => !(CODEX_UNSUPPORTED_BODY_FIELDS as readonly string[]).includes(key))) as P; - -export const stripUnsupportedFields = async ( - ctx: OpenAIResponsesBoundaryCtx, - run: () => Promise, -): Promise => { - ctx.payload = supportedResponsesPayload(ctx.payload); - return await run(); -}; diff --git a/packages/provider-codex/src/interceptors/openai-responses/types.ts b/packages/provider-codex/src/interceptors/openai-responses/types.ts deleted file mode 100644 index a68b1771b7..0000000000 --- a/packages/provider-codex/src/interceptors/openai-responses/types.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import type { ProviderModel, OpenAIResponsesAction } from '@floway-dev/provider'; - -// Boundary ctx for Codex OpenAI Responses interceptors. The same ctx feeds both the -// streaming `/responses` (action='generate') and the non-streaming compaction -// (action='compact') chains; the terminal switches on `action` to pick the -// wire shape (see provider.ts callOpenAIResponses). -export interface OpenAIResponsesBoundaryCtx { - payload: CanonicalOpenAIResponsesPayload; - headers: Headers; - readonly model: ProviderModel; - // Mirrors the gateway-side OpenAIResponsesInvocation.action. Interceptors MAY - // mutate it during the chain to re-route dispatch in the terminal - // handler — the terminal reads `ctx.action`, not the parameter the - // provider was originally called with. - action: OpenAIResponsesAction; -} diff --git a/packages/provider-codex/src/models.ts b/packages/provider-codex/src/models.ts index d844249fe8..0d7ebebdf4 100644 --- a/packages/provider-codex/src/models.ts +++ b/packages/provider-codex/src/models.ts @@ -163,7 +163,7 @@ export const codexModelUsesResponsesLite = (model: Pick): ProviderModel => { const contextWindow = assertContextWindow(raw.context_window, raw.id, 'context_window'); const maxContextWindow = raw.max_context_window === undefined diff --git a/packages/provider-codex/src/pipeline-facts.ts b/packages/provider-codex/src/pipeline-facts.ts index bb443e1fe4..b21860a49c 100644 --- a/packages/provider-codex/src/pipeline-facts.ts +++ b/packages/provider-codex/src/pipeline-facts.ts @@ -1,5 +1,5 @@ import type { CodexPlanObservation } from './access-token.ts'; -import type { CodexCallEffects, CodexBackendCallBase } from './fetch.ts'; +import type { CodexCallEffects, CodexBackendCallBase } from './backend.ts'; import type { CodexAccountCredential, CodexAccessTokenEntry } from './state.ts'; import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; import { secret, type Secret } from '@floway-dev/pipeline'; diff --git a/packages/provider-codex/src/pipelines.ts b/packages/provider-codex/src/pipelines.ts index 5ebd62a93f..08cf17a15f 100644 --- a/packages/provider-codex/src/pipelines.ts +++ b/packages/provider-codex/src/pipelines.ts @@ -5,7 +5,8 @@ import { injectCodexDefaultInstructions } from './stages/inject-default-instruct import { normalizeCodexResponsesHeaders } from './stages/normalize-responses-headers.ts'; import { observeCodexResponse } from './stages/observe-response.ts'; import { prepareCodexRequest } from './stages/prepare-request.ts'; -import { prepareCodexResponses } from './stages/prepare-responses.ts'; +import { prepareCodexResponsesPayload } from './stages/prepare-responses-content.ts'; +import { prepareCodexResponsesHttp } from './stages/prepare-responses.ts'; import { readCodexAccount } from './stages/read-account.ts'; import { restoreCodexResponsesLite } from './stages/restore-responses-lite.ts'; import { restoreCodexResponsesOutput } from './stages/restore-responses-output.ts'; @@ -16,8 +17,8 @@ import { compose } from '@floway-dev/pipeline'; import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction, type ProviderChatResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; export const createCodexPipelines = (config: CodexPipelineConfig): ProviderPipelines => ({ - openaiResponses: compose, ProviderChatResponse<'openaiResponses'>>('codex.openaiResponses', [selectProviderResponsesAction('generate'), readCodexAccount<'openaiResponses'>(config), authenticateCodex(config, 'openaiResponses'), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponses(config, 'openaiResponses'), restoreCodexResponsesOutput, restoreCodexResponsesLite, decodeProviderResponse('openaiResponses'), retryCodexAccess(config, 'openaiResponses'), authorizeCodexHeaders, normalizeCodexResponsesHeaders, observeCodexResponse(config, 'openaiResponses'), observeProviderResponsesCall, http]), - openaiResponsesCompact: compose, ProviderChatResponse<'openaiResponsesCompact'>>('codex.openaiResponsesCompact', [selectProviderResponsesAction('compact'), readCodexAccount<'openaiResponsesCompact'>(config), authenticateCodex(config, 'openaiResponsesCompact'), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponses(config, 'openaiResponsesCompact'), restoreCodexResponsesLite, decodeProviderResponse('openaiResponsesCompact'), retryCodexAccess(config, 'openaiResponsesCompact'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), + openaiResponses: compose, ProviderChatResponse<'openaiResponses'>>('codex.openaiResponses', [selectProviderResponsesAction('generate'), readCodexAccount<'openaiResponses'>(config), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponsesPayload<'openaiResponses'>(), authenticateCodex(config, 'openaiResponses'), prepareCodexResponsesHttp<'openaiResponses'>(), restoreCodexResponsesOutput, restoreCodexResponsesLite, decodeProviderResponse('openaiResponses'), retryCodexAccess(config, 'openaiResponses'), authorizeCodexHeaders, normalizeCodexResponsesHeaders, observeCodexResponse(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderChatResponse<'openaiResponsesCompact'>>('codex.openaiResponsesCompact', [selectProviderResponsesAction('compact'), readCodexAccount<'openaiResponsesCompact'>(config), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponsesPayload<'openaiResponsesCompact'>(), authenticateCodex(config, 'openaiResponsesCompact'), prepareCodexResponsesHttp<'openaiResponsesCompact'>(), restoreCodexResponsesLite, decodeProviderResponse('openaiResponsesCompact'), retryCodexAccess(config, 'openaiResponsesCompact'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), alphaSearch: compose, ProviderResponse>('codex.alphaSearch', [readCodexAccount<'alphaSearch'>(config), authenticateCodex(config, 'alphaSearch'), prepareCodexRequest('alphaSearch'), retryCodexAccess(config, 'alphaSearch'), authorizeCodexHeaders, observeCodexResponse(config, 'alphaSearch'), observeProviderCall, http]), openaiImagesGenerations: compose, ProviderResponse>('codex.openaiImagesGenerations', [readCodexAccount<'openaiImagesGenerations'>(config), authenticateCodex(config, 'openaiImagesGenerations'), prepareCodexRequest('openaiImagesGenerations'), retryCodexAccess(config, 'openaiImagesGenerations'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesGenerations'), observeProviderCall, http]), openaiImagesEdits: compose, ProviderResponse>('codex.openaiImagesEdits', [readCodexAccount<'openaiImagesEdits'>(config), authenticateCodex(config, 'openaiImagesEdits'), prepareCodexRequest('openaiImagesEdits'), retryCodexAccess(config, 'openaiImagesEdits'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesEdits'), observeProviderCall, http]), diff --git a/packages/provider-codex/src/provider.ts b/packages/provider-codex/src/provider.ts index d4367f60c0..73fb81e9d4 100644 --- a/packages/provider-codex/src/provider.ts +++ b/packages/provider-codex/src/provider.ts @@ -1,15 +1,12 @@ import { ensureCodexAccessToken, mintCodexAccessToken } from './access-token.ts'; import { CodexOAuthSessionTerminatedError } from './auth/oauth.ts'; +import type { CodexCallEffects } from './backend.ts'; import { assertCodexUpstreamRecord, type CodexUpstreamConfig } from './config.ts'; import { CODEX_DEFAULT_FLAGS } from './defaults.ts'; -import { callCodexAlphaSearch, callCodexOpenAIImagesEdits, callCodexOpenAIImagesGenerations, callCodexOpenAIResponses, callCodexOpenAIResponsesCompact, type CodexCallEffects } from './fetch.ts'; -import { CODEX_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; -import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { codexImageProviderModel, codexPlanSupportsImages, codexRawToProviderModel, fetchCodexCatalog } from './models.ts'; import { createCodexPipelines } from './pipelines.ts'; import { assertCodexUpstreamState, findCodexAccountIndex, persistCodexRefreshTokenRotation, persistCodexTerminalState } from './state.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import { getProviderRepo, resolveEffectiveFlags, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type UpstreamRecord } from '@floway-dev/provider'; +import { getProviderRepo, resolveEffectiveFlags, type ProviderInstance, type Provider, type UpstreamRecord } from '@floway-dev/provider'; // https://github.com/openai/codex/blob/c607da9f371bb66a41cc772c6ddf1989d28137d3/codex-rs/codex-api/src/requests/headers.rs#L5-L12 // https://github.com/openai/codex/blob/c607da9f371bb66a41cc772c6ddf1989d28137d3/codex-rs/codex-api/src/endpoint/responses.rs#L87-L96 @@ -38,7 +35,7 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { // Computed once per provider instance: only the upstream layer applies // (no per-model override layer). Threaded into every ProviderModel emitted - // by getProvidedModels so interceptors can read the effective flag set + // by getProvidedModels so stages can read the effective flag set // without re-resolving. const enabledFlags = resolveEffectiveFlags([CODEX_DEFAULT_FLAGS, record.flagOverrides]); @@ -100,66 +97,6 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { if (codexPlanSupportsImages(access.planType ?? accountIdentity.planType ?? undefined)) models.push(codexImageProviderModel(enabledFlags)); return models; }, - - callAlphaSearch: async (model, body, signal, opts) => { - const { account } = await readActiveAccount(); - return await callCodexAlphaSearch({ - upstreamId: record.id, - account, - model, - headers: new Headers(opts.headers), - signal, - effects, - call: opts, - body, - }); - }, - - callOpenAIResponses: async (model, body, action, signal, opts) => { - const ctx: OpenAIResponsesBoundaryCtx = { - payload: { ...body, model: model.id }, - headers: new Headers(opts.headers), - model, - action, - }; - return await runInterceptors( - ctx, CODEX_OPENAI_RESPONSES_BOUNDARY, async () => { - const { account } = await readActiveAccount(); - const { model: _ignored, ...wireBody } = ctx.payload; - const backendCallBase = { upstreamId: record.id, account, model, headers: ctx.headers, signal, effects, call: opts }; - switch (ctx.action) { - case 'compact': - // The fetch boundary selects the private wire format before compact - // projection so interceptor-provided tools and instructions survive. - return { action: 'compact', ...(await callCodexOpenAIResponsesCompact({ ...backendCallBase, body: wireBody })) }; - case 'generate': - return { action: 'generate', ...(await callCodexOpenAIResponses({ ...backendCallBase, body: wireBody })) }; - default: - ctx.action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${ctx.action as string}`); - } - }, - ); - }, - - // Codex exposes OpenAI Responses and its provider-owned image endpoints. The - // remaining surfaces are unreachable through the advertised catalog, but - // a stray dispatch must still surface as a structured 405. - callAnthropicMessages: () => unsupportedStreamResult(), - callAnthropicMessagesCountTokens: () => unsupportedCallResult(), - callOpenAICompletions: () => unsupportedCallResult(), - callOpenAIChatCompletions: () => unsupportedStreamResult(), - callOpenAIEmbeddings: () => unsupportedCallResult(), - callOpenAIImagesGenerations: async (model, body, signal, opts) => { - const { account } = await readActiveAccount(); - return await callCodexOpenAIImagesGenerations({ upstreamId: record.id, account, model, headers: opts.headers, signal, effects, call: opts, body, fallbackPlanType: accountIdentity.planType ?? undefined }); - }, - callOpenAIImagesEdits: async (model, request, signal, opts) => { - const { account } = await readActiveAccount(); - return await callCodexOpenAIImagesEdits({ upstreamId: record.id, account, model, headers: opts.headers, signal, effects, call: opts, request, fallbackPlanType: accountIdentity.planType ?? undefined }); - }, - callOpenAIAudioTranscriptions: () => unsupportedCallResult(), - callRerank: () => Promise.reject(new Error('Codex provider does not support callRerank')), }; return { @@ -174,14 +111,3 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { instance, }; }; - -const synthetic405 = (): Response => new Response( - JSON.stringify({ error: { type: 'method_not_allowed', message: 'Endpoint not supported by codex provider' } }), - { status: 405, headers: { 'content-type': 'application/json' } }, -); - -const unsupportedStreamResult = (): Promise> => - Promise.resolve({ ok: false, modelKey: '', response: synthetic405() }); - -const unsupportedCallResult = (): Promise => - Promise.resolve({ modelKey: '', response: synthetic405() }); diff --git a/packages/provider-codex/src/stages/authenticate.ts b/packages/provider-codex/src/stages/authenticate.ts index d7f192d627..c47fdb6db3 100644 --- a/packages/provider-codex/src/stages/authenticate.ts +++ b/packages/provider-codex/src/stages/authenticate.ts @@ -1,5 +1,5 @@ import { codexPlanObservation } from '../access-token.ts'; -import { prepareCodexCall } from '../fetch.ts'; +import { prepareCodexCall } from '../backend.ts'; import { codexPlanSupportsImages } from '../models.ts'; import { codexCall, tokenFacts, type CodexAccountFacts, type CodexAuthenticatedFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; import { takeHttpResponse } from '@floway-dev/http/pipeline'; diff --git a/packages/provider-codex/src/stages/inject-default-instructions.ts b/packages/provider-codex/src/stages/inject-default-instructions.ts index 3e09c48bb8..b80c415c6e 100644 --- a/packages/provider-codex/src/stages/inject-default-instructions.ts +++ b/packages/provider-codex/src/stages/inject-default-instructions.ts @@ -1,10 +1,21 @@ -import { defaultInstructionsPayload } from '../interceptors/openai-responses/inject-default-instructions.ts'; import type { CodexAccountFacts } from '../pipeline-facts.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; type Request = CodexAccountFacts<'openaiResponses' | 'openaiResponsesCompact'>; +// ChatGPT-subscription catalog models reject missing or empty `instructions`. +// Native and translated callers may omit the field, so the provider supplies a +// neutral value at its boundary. Other values remain upstream-owned validation. +// https://github.com/im4codes/imcodes/blob/5f769d933dfd679e3a4d670183b0384a1baf62cd/src/agent/providers/codex-sdk.ts#L560-L579 +export const defaultInstructionsPayload =

>(payload: P): P => { + const instructions = payload.instructions; + return instructions === undefined || instructions === null || instructions === '' + ? { ...payload, instructions: "You're a helpful assistant." } + : payload; +}; + export const injectCodexDefaultInstructions = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ name: 'injectCodexDefaultInstructions', through: { diff --git a/packages/provider-codex/src/stages/observe-response.ts b/packages/provider-codex/src/stages/observe-response.ts index cd2d005dbd..f0ee9d5063 100644 --- a/packages/provider-codex/src/stages/observe-response.ts +++ b/packages/provider-codex/src/stages/observe-response.ts @@ -1,4 +1,4 @@ -import { classifyCodexUnauthorizedResponse, decodeCodexUpstreamError, writeCodexQuotaObservation } from '../fetch.ts'; +import { classifyCodexUnauthorizedResponse, decodeCodexUpstreamError, writeCodexQuotaObservation } from '../backend.ts'; import { codexCall, type CodexHttpFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; import { exchangeResponse, takeHttpResponse, type HttpResponseFacts } from '@floway-dev/http/pipeline'; import { defer, defineStage, move, setRelease, type Deferred } from '@floway-dev/pipeline'; diff --git a/packages/provider-codex/src/stages/prepare-responses-content.ts b/packages/provider-codex/src/stages/prepare-responses-content.ts new file mode 100644 index 0000000000..9bcc4d18fe --- /dev/null +++ b/packages/provider-codex/src/stages/prepare-responses-content.ts @@ -0,0 +1,15 @@ +import { prepareCodexResponsesContent, type PreparedCodexResponsesRequest } from '../backend.ts'; +import { accountValue, type CodexAccountFacts } from '../pipeline-facts.ts'; +import { defineStage, isSecret, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export interface CodexPreparedResponsesFacts { 'request.codex.responsesPrepared': PreparedCodexResponsesRequest } +export const prepareCodexResponsesPayload = () => defineStage & { 'request.provider.responsesAction': 'generate' | 'compact' }, CodexAccountFacts & CodexPreparedResponsesFacts, ProviderChatResponse, ProviderChatResponse>({ + name: 'prepareCodexResponsesPayload', + through: { request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.codex.account', 'request.provider.responsesAction'], consumes: [], provides: ['request.codex.responsesPrepared'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => { + const headers = new Headers(facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value])); + const prepared = prepareCodexResponsesContent({ account: accountValue(facts['request.codex.account']), model: facts['request.provider.model'], body: facts['request.provider.payload'], headers }, facts['request.provider.responsesAction']); + return move({ ...await next(move({ ...facts, 'request.codex.responsesPrepared': prepared })) }); + }, +}); diff --git a/packages/provider-codex/src/stages/prepare-responses.ts b/packages/provider-codex/src/stages/prepare-responses.ts index 341eeb1acd..5875841c08 100644 --- a/packages/provider-codex/src/stages/prepare-responses.ts +++ b/packages/provider-codex/src/stages/prepare-responses.ts @@ -1,8 +1,8 @@ +import type { CodexPreparedResponsesFacts } from './prepare-responses-content.ts'; import { CODEX_BACKEND_BASE, CODEX_CLI_VERSION, CODEX_OPENAI_RESPONSES_COMPACT_PATH, CODEX_OPENAI_RESPONSES_PATH, CODEX_ORIGINATOR, CODEX_RESPONSES_LITE_HEADER, CODEX_USER_AGENT } from '../constants.ts'; -import { prepareCodexResponsesContent } from '../fetch.ts'; -import { codexCall, type CodexAuthenticatedFacts, type CodexHttpFacts, type CodexPipelineConfig } from '../pipeline-facts.ts'; +import { type CodexAuthenticatedFacts, type CodexHttpFacts } from '../pipeline-facts.ts'; import type { CodexResponsesLiteRequest } from '../responses-lite.ts'; -import { defineStage, isSecret, move } from '@floway-dev/pipeline'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; export interface CodexResponsesHttpFacts extends CodexHttpFacts { @@ -10,21 +10,19 @@ export interface CodexResponsesHttpFacts extends CodexHttpFacts { 'request.codex.responsesLite': CodexResponsesLiteRequest | null; } -export const prepareCodexResponses = (config: CodexPipelineConfig, _operation: O) => defineStage & Pick, CodexResponsesHttpFacts, ProviderChatResponse, ProviderChatResponse, ProviderChatServices>({ - name: 'prepareCodexResponses', +export const prepareCodexResponsesHttp = () => defineStage & Pick & CodexPreparedResponsesFacts, CodexResponsesHttpFacts, ProviderChatResponse, ProviderChatResponse, ProviderChatServices>({ + name: 'prepareCodexResponsesHttp', through: { request: { - needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.codex.account', 'request.codex.accessToken', 'request.codex.plan', 'request.provider.responsesAction'], - consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], + needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.codex.account', 'request.codex.accessToken', 'request.codex.plan', 'request.provider.responsesAction', 'request.codex.responsesPrepared'], + consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.codex.responsesPrepared'], provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.codex.model', 'request.codex.modelKey', 'request.provider.modelKey', 'request.provider.responsesAction', 'request.codex.responsesLite'], }, response: { needs: [], consumes: [], provides: [] }, }, - execute: async (facts, next, use) => { - const { 'request.provider.model': model, 'request.provider.payload': body, ...rest } = facts; + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': _body, 'request.codex.responsesPrepared': prepared, ...rest } = facts; const action = facts['request.provider.responsesAction']; - const inbound = new Headers(facts['request.http.headers'].map(([name, value]) => [name, isSecret(value) ? value.reveal() : value])); - const prepared = prepareCodexResponsesContent({ ...codexCall(config, facts, use, model), headers: inbound, body }, action); const { identity } = prepared; const headers: [string, string][] = [ ['originator', CODEX_ORIGINATOR], ['user-agent', CODEX_USER_AGENT], ['version', CODEX_CLI_VERSION], diff --git a/packages/provider-codex/src/stages/retry-access.ts b/packages/provider-codex/src/stages/retry-access.ts index bd8cd0662d..137e34f642 100644 --- a/packages/provider-codex/src/stages/retry-access.ts +++ b/packages/provider-codex/src/stages/retry-access.ts @@ -1,5 +1,5 @@ import { codexPlanObservation } from '../access-token.ts'; -import { refreshAccessTokenForRetry } from '../fetch.ts'; +import { refreshAccessTokenForRetry } from '../backend.ts'; import { codexPlanSupportsImages } from '../models.ts'; import { codexCall, tokenFacts, type CodexHttpFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; import { takeHttpResponse, type HttpResponseFacts } from '@floway-dev/http/pipeline'; diff --git a/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts b/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts index 75fb8e4b1e..95bb65ee80 100644 --- a/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts +++ b/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts @@ -1,10 +1,33 @@ -import { supportedResponsesPayload } from '../interceptors/openai-responses/strip-unsupported-fields.ts'; import type { CodexAccountFacts } from '../pipeline-facts.ts'; import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; type Request = CodexAccountFacts<'openaiResponses' | 'openaiResponsesCompact'>; +// Codex backend rejects requests carrying any of these fields with a +// `Unsupported parameter: ` 4xx. They are regular OpenAI Responses API +// fields that the ChatGPT-subscription path does not honor. Source-protocol +// translators legitimately set max_output_tokens / temperature / top_p (the +// caller's request might carry them), so we strip them at the Codex target +// boundary rather than at translation time, where they remain valid for +// other providers. +const CODEX_UNSUPPORTED_BODY_FIELDS = [ + 'max_output_tokens', + 'temperature', + 'top_p', + 'frequency_penalty', + 'presence_penalty', + 'user', + 'metadata', + 'prompt_cache_retention', + 'safety_identifier', + 'stream_options', +] as const; + +export const supportedResponsesPayload =

>(payload: P): P => + Object.fromEntries(Object.entries(payload).filter(([key]) => !(CODEX_UNSUPPORTED_BODY_FIELDS as readonly string[]).includes(key))) as P; + export const stripCodexUnsupportedFields = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ name: 'stripCodexUnsupportedFields', through: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b676cd3935..0ae59e6103 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -533,9 +533,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline @@ -561,9 +558,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline From c32f0781e465bbf1c8fc9234a9596d13e0cbc5c6 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:51:02 +0800 Subject: [PATCH 129/156] refactor: finish provider pipeline dependency cleanup Drop the remaining Azure and Copilot dependency edges to the retired private-chain framework. Keep shared provider comments aligned with fact handoffs, stage planning and the catalog-only instance surface. --- packages/provider/src/error.ts | 2 +- packages/provider/src/flags.ts | 4 +-- packages/provider/src/invocation.ts | 47 ++++------------------------- packages/provider/src/options.ts | 3 +- packages/provider/src/result.ts | 4 +-- pnpm-lock.yaml | 6 ---- 6 files changed, 12 insertions(+), 54 deletions(-) diff --git a/packages/provider/src/error.ts b/packages/provider/src/error.ts index 3fe5498872..1ccfd6ee9a 100644 --- a/packages/provider/src/error.ts +++ b/packages/provider/src/error.ts @@ -1,4 +1,4 @@ -// Errors that bubble out of source/target emit or interceptors and need a +// Errors that bubble out of source/target emit or stages and need a // structured envelope for the api debug response. The target_api lane is // typed as a free string here so the package stays decoupled from the // api-internal serve-api unions — the api always passes the narrowed value diff --git a/packages/provider/src/flags.ts b/packages/provider/src/flags.ts index 9839c52bd6..c34b4b2fe8 100644 --- a/packages/provider/src/flags.ts +++ b/packages/provider/src/flags.ts @@ -1,11 +1,11 @@ // Stable identities for every admin-toggleable per-upstream behavior flag. -// Interceptor code references a flag by id; the dependency goes interceptor +// Stage code references a flag by id; the dependency goes stage // → flag, never the other way. The dashboard owns presentation and translated // copy, while this package owns the ids shared by providers and persisted // overrides. // // Vendor-style flags (`vendor-deepseek`, `vendor-qwen`, `vendor-kimi`) are -// mutually exclusive per model: a vendor interceptor translates the +// mutually exclusive per model: a vendor stage translates the // gateway's OpenAI-canonical request and response shape into the vendor's // wire dialect; with no vendor flag set, behavior defaults to the OpenAI // standard and no vendor rewrite runs. diff --git a/packages/provider/src/invocation.ts b/packages/provider/src/invocation.ts index 081c9e8a48..cb7d1b9224 100644 --- a/packages/provider/src/invocation.ts +++ b/packages/provider/src/invocation.ts @@ -9,23 +9,8 @@ import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/open export type ChatTargetApi = 'anthropicMessages' | 'openaiResponses' | 'openaiChatCompletions'; -// One (provider, model) pair the resolver produced for an inbound id, -// plus the per-request `Fetcher` minted for the provider's upstream. The -// pair is the smallest unit the dispatch layer needs to make a wire call: -// `provider.instance.callXxx(providerModelOf(candidate), body, ...)` — -// upstream id / upstream name / provider kind / capability flags come off -// `provider.*`, the merged public metadata (id, endpoints, limits, ...) off -// `model.*`, and the per-upstream `ProviderModel` (providerData, -// enabledFlags) off `providerModelOf(candidate)`. -// -// Resolution narrows by `model.kind` only — choosing the inbound target -// protocol from `model.endpoints` is the attempt layer's job, not part of -// the candidate. -// -// `rules` is set only for candidates minted by the alias walk — it carries -// the picked target's rule overlay so the attempt's terminal wire call can -// apply it against the target IR. Absent (undefined) for direct-resolution -// candidates and present (possibly `{}`) for alias-origin candidates. +// A resolver's live candidate belongs to services. Pipeline facts retain its identifier and +// portable model/rule content; the selected operation hands off into provider.pipelines. export interface ModelCandidate { readonly provider: Provider; readonly model: InternalModel; @@ -33,19 +18,7 @@ export interface ModelCandidate { readonly rules?: AliasRules; } -// Pull the emitting upstream's `ProviderModel` off the candidate. Dispatch -// hands this to the provider's `callXxx`; interceptor gates read -// `.enabledFlags`, boundary shims read `.providerData`, etc. The candidate -// always names exactly one upstream via `provider.upstreamId`; for real-row -// candidates the resolver populates `model.providerModels` with an entry -// under that key at candidate-creation time. -// -// Two error paths, distinguished so a caller reading the message can tell -// which invariant broke: an alias row was mistakenly used as a dispatch -// target (the resolver should have expanded it to its target's real row -// first), or a real row is missing the entry for the candidate's upstream -// (the candidate was assembled outside the resolver, or the row was merged -// after the upstream stopped contributing). +// Alias rows must be expanded before selecting one upstream's catalog metadata. export const providerModelOf = (candidate: ModelCandidate): ProviderModel => { const { model, provider } = candidate; if (model.providerModels === undefined) { @@ -58,13 +31,8 @@ export const providerModelOf = (candidate: ModelCandidate): ProviderModel => { return providerModel; }; -// Per-protocol invocation shape passed to interceptors. Carries the -// source-shape request body (mutable, so the body can be cleaned), the -// candidate the attempt is dispatching against, the chat target protocol -// the attempt picked for this candidate, and a mutable `Headers` instance -// carried into the boundary chain — so workarounds that only need to set -// or drop a header stay at the owning interceptor boundary instead of -// widening the provider call signature. +// Local planning views carry immutable payload content and live candidate/transport handles. +// Stages materialize their resulting payload, action and header lines into facts. export interface AnthropicMessagesInvocation { payload: AnthropicMessagesPayload; readonly candidate: ModelCandidate; @@ -74,10 +42,7 @@ export interface AnthropicMessagesInvocation { export interface OpenAIResponsesInvocation { payload: CanonicalOpenAIResponsesPayload; - // Mutable action tag — interceptors can flip 'compact' to 'generate' so the - // inner provider call runs a normal summarization turn (see the - // openai-responses-compact-shim) and the gateway derives snapshot mode from the - // post-chain action carried on the provider's tagged result. + // Planning may choose a different dispatched action while preserving source intent. action: OpenAIResponsesAction; readonly candidate: ModelCandidate; readonly targetApi: ChatTargetApi; diff --git a/packages/provider/src/options.ts b/packages/provider/src/options.ts index b98d35ad18..20251c0e79 100644 --- a/packages/provider/src/options.ts +++ b/packages/provider/src/options.ts @@ -21,8 +21,7 @@ export interface UpstreamFetchOptions { } // Identity wrapper for callers that don't participate in per-request TTFT -// timing — model-listing helpers and interceptor sub-calls that dispatch -// outside the primary data-plane fetch. +// timing — catalog and control calls outside the primary model-endpoint timing window. export const identityWrapUpstreamCall = (dispatch: () => Promise): Promise => dispatch(); // Transfer a request into the fetcher exactly once. The wrapper used for diff --git a/packages/provider/src/result.ts b/packages/provider/src/result.ts index ba8c445446..79c1ebea95 100644 --- a/packages/provider/src/result.ts +++ b/packages/provider/src/result.ts @@ -41,7 +41,7 @@ export interface ApiErrorResult { upstreamId?: string; } -// Gateway-side bug surface (parser crash, interceptor throw, etc.). The +// Gateway-side bug surface (parser crash, stage throw, etc.). The // protocol's respond layer renders a debug envelope around `error` // (stack, cause, target_api) rather than passing through a wire body — // the shape differs from `ApiErrorResult` for that reason. @@ -54,7 +54,7 @@ export interface InternalErrorResult { // A fully-shaped non-streaming success body — the output of a source endpoint // that measures rather than generates (count_tokens). It is NOT an -// `ExecuteResult`: the target emit/interceptor layer never produces one. The +// `ExecuteResult`: the target emit/stage layer never produces one. The // orchestrator passes it straight to `respond` without persistence, and // `respond` emits it verbatim. `upstreamId` is present when the body came from // a real upstream call and absent for gateway-synthesized envelopes (rewrite diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0ae59e6103..39285d7a0e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -511,9 +511,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline @@ -580,9 +577,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline From 5ca0f6719a187d4f5f5b7b7e97e204d301fc2bf9 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 06:58:27 +0800 Subject: [PATCH 130/156] fix: preserve reported zero chat token quantities Keep input and output presence through native usage snapshots and pricing measurements. A reported zero stays a zero quantity; missing counters and their pricing selector remain absent. Preserve the existing cache normalization. All 74 chat files and 1127 tests pass, including native zero, partial Messages accounting, and actual Custom HTTP wire measurements. --- .../answer-websocket-warmup_test.ts | 4 +- .../data-plane/chat/provider-handoffs_test.ts | 11 ++++++ .../data-plane/chat/shared/usage_test.ts | 37 +++++++++++++++++++ .../chat/anthropic-messages/meter.ts | 13 +++---- .../chat/anthropic-messages/usage.ts | 11 +++--- .../chat/openai-chat-completions/meter.ts | 13 +++---- .../chat/openai-chat-completions/usage.ts | 11 +++--- .../data-plane/chat/openai-responses/meter.ts | 10 ++--- .../data-plane/chat/openai-responses/usage.ts | 11 +++--- .../src/data-plane/chat/shared/usage.ts | 24 ++++++++++++ 10 files changed, 109 insertions(+), 36 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/shared/usage_test.ts create mode 100644 packages/gateway/src/data-plane/chat/shared/usage.ts diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts index d1fe189b98..de8bf7f280 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts @@ -14,10 +14,10 @@ const exercise = async (generate: boolean | null | undefined) => { const dial = vi.fn(); const ending = defineStage, Record>({ name: 'scriptedWarmupDial', - return: { provides: ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers'] }, + return: { provides: ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.body'] }, execute: async facts => { dial(); - return move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream', frames: (async function* () { yield doneFrame(); })() }, 'response.chat.openaiResponses.streamedUsage': null, 'response.usage.billable': [], 'response.http.headers': [] }); + return move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream', frames: (async function* () { yield doneFrame(); })() }, 'response.chat.openaiResponses.streamedUsage': null, 'response.usage.billable': [], 'response.http.headers': [], 'response.http.body': null }); }, }); const payload: CanonicalOpenAIResponsesPayload = { model: 'client-model', input: [{ type: 'message', role: 'developer', content: 'Base instructions' }], ...(generate === undefined ? {} : { generate }) }; diff --git a/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts b/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts index f3370ea5ac..841b476a0b 100644 --- a/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts @@ -93,3 +93,14 @@ test('an actual upstream quantity-formatting fault rejects produced usage with t expect(causes.length).toBeGreaterThan(0); expect(causes.every(error => error === original)).toBe(true); }); + +test('actual native HTTP zero usage remains measured after the complete wire and client projection', async () => { + const zero = { ...usage, usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 } }; + const { executed } = await runWire(new Response([first, zero].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } })); + const iterator = (executed.facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable)[Symbol.asyncIterator](); + let step = await iterator.next(); + while (!step.done) step = await iterator.next(); + await executed.drain(); + const outcome = await (executed.facts['response.chat.openaiChatCompletions.streamedUsage'] as Deferred); + expect(outcome).toMatchObject({ failed: false, billable: [{ quantities: { input_tokens: '0', output_tokens: '0' }, pricingFacts: { inputTokens: 0 } }] }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/usage_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/usage_test.ts new file mode 100644 index 0000000000..1a171e30cf --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/shared/usage_test.ts @@ -0,0 +1,37 @@ +import { expect, test } from 'vitest'; + +import { createAnthropicMessagesBillableUsageReader } from '../../../../src/data-plane/chat/anthropic-messages/usage.ts'; +import { billableUsageFromOpenAIChatCompletionsEvent } from '../../../../src/data-plane/chat/openai-chat-completions/usage.ts'; +import { billableUsageFromOpenAIResponsesResult } from '../../../../src/data-plane/chat/openai-responses/usage.ts'; +import { chatUsageMeasurement } from '../../../../src/data-plane/chat/shared/usage.ts'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; + +for (const protocol of ['openaiChatCompletions', 'anthropicMessages', 'openaiResponses'] as const) test(`${protocol} preserves reported zero input and output as measured quantities`, () => { + const usage = protocol === 'openaiChatCompletions' + ? billableUsageFromOpenAIChatCompletionsEvent({ id: 'c', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 } }) + : protocol === 'openaiResponses' + ? billableUsageFromOpenAIResponsesResult({ usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 } }) + : createAnthropicMessagesBillableUsageReader()({ type: 'message_start', message: { id: 'm', type: 'message', role: 'assistant', model: 'm', content: [], stop_reason: null, stop_sequence: null, usage: { input_tokens: 0, output_tokens: 0 } } }); + if (usage === null) throw new Error('A reported zero usage block must remain a measurement'); + expect(chatUsageMeasurement(usage)).toEqual({ quantities: { input_tokens: '0', output_tokens: '0' }, pricingFacts: { inputTokens: 0 }, dumpTokenUsage: { input: 0, output: 0 } }); +}); + +test('absent native usage remains absent rather than reporting zero', () => { + const chunk: OpenAIChatCompletionsStreamEvent = { id: 'c', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [] }; + expect(billableUsageFromOpenAIChatCompletionsEvent(chunk)).toBeNull(); + expect(billableUsageFromOpenAIResponsesResult({})).toBeNull(); + expect(createAnthropicMessagesBillableUsageReader()({ type: 'message_stop' })).toBeNull(); +}); + +test('Messages partial input accounting preserves zero without manufacturing unreported output', () => { + const usage = createAnthropicMessagesBillableUsageReader()({ type: 'message_start', message: { usage: { input_tokens: 0 } } } as AnthropicMessagesStreamEvent); + if (usage === null) throw new Error('Reported input must remain measurable'); + expect(chatUsageMeasurement(usage)).toEqual({ quantities: { input_tokens: '0' }, pricingFacts: { inputTokens: 0 }, dumpTokenUsage: { input: 0 } }); +}); + +test('Messages partial output accounting preserves zero without manufacturing input or its pricing selector', () => { + const usage = createAnthropicMessagesBillableUsageReader()({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 0 } }); + if (usage === null) throw new Error('Reported output must remain measurable'); + expect(chatUsageMeasurement(usage)).toEqual({ quantities: { output_tokens: '0' }, pricingFacts: {}, dumpTokenUsage: { output: 0 } }); +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts index ceb42d4ffe..0c89f2bc81 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts @@ -2,11 +2,11 @@ import type { StreamOutcome } from '../../pipeline/serve.ts'; import { meterChatWire } from '../meter.ts'; import { createAnthropicMessagesBillableUsageReader } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; -import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; +import { chatUsageMeasurement, type ChatBillableUsage } from '../shared/usage.ts'; import { defer, type Deferred } from '@floway-dev/pipeline'; import { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame, BillableUsage } from '@floway-dev/protocols/common'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; export const meterUsage = (streamedUsage: string) => meterChatWire({ @@ -34,7 +34,7 @@ const meterAnthropicMessages = ( let failed = false; const readBillableUsage = createAnthropicMessagesBillableUsageReader(); const generator = (async function* () { - let reported: BillableUsage | undefined; + let reported: ChatBillableUsage | undefined; const finish = (): void => { if (settled) return; settled = true; @@ -88,9 +88,8 @@ const isAnthropicMessagesTerminalFrame = (frame: ProtocolFrame { - const tokens = tokenUsageFromBillableUsage(usage); - if (tokens === null) return { identity, quantities: {} }; - const measurement = tokenUsageMeasurement(tokens); +const billedEntity = (usage: ChatBillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { + if (usage === undefined) return { identity, quantities: {} }; + const measurement = chatUsageMeasurement(usage); return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; }; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/usage.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/usage.ts index d0dbcaab14..d62790b6fb 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/usage.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/usage.ts @@ -1,25 +1,26 @@ +import type { ChatBillableUsage } from '../shared/usage.ts'; import { mergeAnthropicMessagesUsageSnapshot, anthropicMessagesUsageSnapshot, splitAnthropicMessagesCacheCreationTokens, type AnthropicMessagesStreamEvent, type AnthropicMessagesUsageSnapshot } from '@floway-dev/protocols/anthropic-messages'; -import { billableServiceTier, type BillableUsage } from '@floway-dev/protocols/common'; +import { billableServiceTier } from '@floway-dev/protocols/common'; // Anthropic reports `input_tokens` exclusive of both cache buckets already, // and splits cache creation by TTL — the two rates we are billed at. -export const billableUsageFromAnthropicMessagesUsage = (usage: AnthropicMessagesUsageSnapshot): BillableUsage | null => { +export const billableUsageFromAnthropicMessagesUsage = (usage: AnthropicMessagesUsageSnapshot): ChatBillableUsage | null => { if (usage.input_tokens === undefined && usage.output_tokens === undefined) return null; const { cacheWrite, cacheWrite1h } = splitAnthropicMessagesCacheCreationTokens(usage); const tier = billableServiceTier(usage.speed) ?? billableServiceTier(usage.service_tier); return { - input: usage.input_tokens ?? 0, + ...(usage.input_tokens === undefined ? {} : { input: usage.input_tokens }), cacheRead: usage.cache_read_input_tokens ?? 0, cacheWrite, cacheWrite1h, - output: usage.output_tokens ?? 0, + ...(usage.output_tokens === undefined ? {} : { output: usage.output_tokens }), ...(tier !== null ? { tier } : {}), }; }; // Anthropic reports input accounting on `message_start` and output accounting // on `message_delta`, so the running figure is merged across both. -export const createAnthropicMessagesBillableUsageReader = (): (event: AnthropicMessagesStreamEvent) => BillableUsage | null => { +export const createAnthropicMessagesBillableUsageReader = (): (event: AnthropicMessagesStreamEvent) => ChatBillableUsage | null => { let merged = anthropicMessagesUsageSnapshot(); return event => { const usage = event.type === 'message_start' ? event.message.usage diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts index 732905f28f..f089199f65 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts @@ -2,10 +2,10 @@ import type { StreamOutcome } from '../../pipeline/serve.ts'; import { meterChatWire } from '../meter.ts'; import { billableUsageFromOpenAIChatCompletionsEvent } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; -import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../shared/telemetry/usage.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; +import { chatUsageMeasurement, type ChatBillableUsage } from '../shared/usage.ts'; import { defer, type Deferred } from '@floway-dev/pipeline'; -import type { ProtocolFrame, BillableUsage } from '@floway-dev/protocols/common'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { openaiChatCompletionsErrorPayloadMessage, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; @@ -31,7 +31,7 @@ const meterOpenAIChatCompletions = ( let completed = false; let failed = false; const generator = (async function* () { - let reported: BillableUsage | undefined; + let reported: ChatBillableUsage | undefined; const finish = (): void => { if (settled) return; settled = true; @@ -81,9 +81,8 @@ const isTerminal = (frame: ProtocolFrame): boo * at all, which is a different statement from reporting zero — and a rate can depend on the * service tier and on how much input there was, so both travel as pricing facts rather than * being folded into the quantities. */ -const billedEntity = (usage: BillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { - const tokens = tokenUsageFromBillableUsage(usage); - if (tokens === null) return { identity, quantities: {} }; - const measurement = tokenUsageMeasurement(tokens); +const billedEntity = (usage: ChatBillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { + if (usage === undefined) return { identity, quantities: {} }; + const measurement = chatUsageMeasurement(usage); return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/usage.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/usage.ts index e44ba4c97f..136a59ee34 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/usage.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/usage.ts @@ -1,4 +1,5 @@ -import { billableServiceTier, splitInclusiveInputTokens, type BillableUsage } from '@floway-dev/protocols/common'; +import type { ChatBillableUsage } from '../shared/usage.ts'; +import { billableServiceTier, splitInclusiveInputTokens } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; type OpenAIChatCompletionsUsage = NonNullable; @@ -6,7 +7,7 @@ type OpenAIChatCompletionsUsage = NonNullable { +): ChatBillableUsage => { const cacheWrite = usage.prompt_tokens_details?.cache_creation_input_tokens ?? usage.prompt_tokens_details?.cache_write_tokens ?? 0; @@ -17,15 +18,15 @@ export const billableUsageFromOpenAIChatCompletionsUsage = ( ); const tier = billableServiceTier(serviceTier); return { - input, + ...(usage.prompt_tokens === undefined ? {} : { input }), cacheRead, cacheWrite, // OpenAI Chat Completions has no cache-write TTL split. cacheWrite1h: 0, - output: usage.completion_tokens, + ...(usage.completion_tokens === undefined ? {} : { output: usage.completion_tokens }), ...(tier !== null ? { tier } : {}), }; }; -export const billableUsageFromOpenAIChatCompletionsEvent = (event: OpenAIChatCompletionsStreamEvent): BillableUsage | null => +export const billableUsageFromOpenAIChatCompletionsEvent = (event: OpenAIChatCompletionsStreamEvent): ChatBillableUsage | null => event.usage ? billableUsageFromOpenAIChatCompletionsUsage(event.usage, event.service_tier) : null; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/meter.ts b/packages/gateway/src/data-plane/chat/openai-responses/meter.ts index 9b3b490e6e..05e18bc993 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/meter.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/meter.ts @@ -2,10 +2,10 @@ import type { StreamOutcome } from '../../pipeline/serve.ts'; import { meterChatWire } from '../meter.ts'; import { billableUsageFromOpenAIResponsesEvent } from './usage.ts'; import type { BillableEntity } from '../../pipeline/facts.ts'; -import { tokenUsageMeasurement, tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; +import { chatUsageMeasurement, type ChatBillableUsage } from '../shared/usage.ts'; import { defer, type Deferred } from '@floway-dev/pipeline'; -import type { ProtocolFrame, BillableUsage } from '@floway-dev/protocols/common'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { isOpenAIResponsesTerminalEvent, OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; @@ -33,7 +33,7 @@ const meterOpenAIResponses = ( let sawTerminal = false; let failed = false; const generator = (async function* () { - let reported: BillableUsage | undefined; + let reported: ChatBillableUsage | undefined; const finish = (): void => { if (settled) return; settled = true; @@ -92,8 +92,8 @@ const meterOpenAIResponses = ( * because on this protocol it is not a quantity but a rate selector — `service_tier` states * the tier the turn was actually served at, and that is the pricing entry it is billed * under. */ -export const billedOpenAIResponsesEntity = (identity: TelemetryModelIdentity, usage: BillableUsage | undefined): BillableEntity => { +export const billedOpenAIResponsesEntity = (identity: TelemetryModelIdentity, usage: ChatBillableUsage | undefined): BillableEntity => { if (usage === undefined) return { identity, quantities: {} }; - const measurement = tokenUsageMeasurement(tokenUsageFromBillableUsage(usage)); + const measurement = chatUsageMeasurement(usage); return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/usage.ts b/packages/gateway/src/data-plane/chat/openai-responses/usage.ts index 0610b04f17..3280831519 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/usage.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/usage.ts @@ -1,4 +1,5 @@ -import { billableServiceTier, splitInclusiveInputTokens, type BillableUsage } from '@floway-dev/protocols/common'; +import type { ChatBillableUsage } from '../shared/usage.ts'; +import { billableServiceTier, splitInclusiveInputTokens } from '@floway-dev/protocols/common'; import { openaiResponsesResultFromStreamEvent, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; // service_tier reports the tier actually served and therefore selects the @@ -8,7 +9,7 @@ import { openaiResponsesResultFromStreamEvent, type OpenAIResponsesResult, type // body can be priced through the same helper. export const billableUsageFromOpenAIResponsesResult = ( response: { readonly usage?: OpenAIResponsesResult['usage']; readonly service_tier?: OpenAIResponsesResult['service_tier'] }, -): BillableUsage | null => { +): ChatBillableUsage | null => { const usage = response.usage; if (!usage) return null; const cacheWrite = usage.input_tokens_details?.cache_write_tokens ?? 0; @@ -19,17 +20,17 @@ export const billableUsageFromOpenAIResponsesResult = ( ); const tier = billableServiceTier(response.service_tier); return { - input, + ...(usage.input_tokens === undefined ? {} : { input }), cacheRead, cacheWrite, // OpenAI Responses has no cache-write TTL split; every write bills at one rate. cacheWrite1h: 0, - output: usage.output_tokens, + ...(usage.output_tokens === undefined ? {} : { output: usage.output_tokens }), ...(tier !== null ? { tier } : {}), }; }; -export const billableUsageFromOpenAIResponsesEvent = (event: OpenAIResponsesStreamEvent): BillableUsage | null => { +export const billableUsageFromOpenAIResponsesEvent = (event: OpenAIResponsesStreamEvent): ChatBillableUsage | null => { const response = openaiResponsesResultFromStreamEvent(event); return response === null ? null : billableUsageFromOpenAIResponsesResult(response); }; diff --git a/packages/gateway/src/data-plane/chat/shared/usage.ts b/packages/gateway/src/data-plane/chat/shared/usage.ts new file mode 100644 index 0000000000..003ded5773 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/usage.ts @@ -0,0 +1,24 @@ +import type { TokenUsage } from '../../../repo/types.ts'; +import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; +import { tokenUsage } from '../../shared/telemetry/usage.ts'; +import type { UsageMeasurement } from '../../shared/telemetry/usage.ts'; +import type { BillableUsage } from '@floway-dev/protocols/common'; + +export type ChatBillableUsage = Omit & { readonly input?: number; readonly output?: number }; + +export const chatUsageMeasurement = (usage: ChatBillableUsage): UsageMeasurement => { + const tokens: TokenUsage = { + ...tokenUsage({ input_cache_read: usage.cacheRead, input_cache_write: usage.cacheWrite, input_cache_write_1h: usage.cacheWrite1h }), + ...(usage.input === undefined ? {} : { input: usage.input }), + ...(usage.output === undefined ? {} : { output: usage.output }), + ...(usage.tier === undefined ? {} : { tier: usage.tier }), + }; + return { + quantities: tokenUsageQuantities(tokens), + pricingFacts: { + ...(usage.tier === undefined ? {} : { serviceTier: usage.tier }), + ...(usage.input === undefined ? {} : { inputTokens: usage.input + usage.cacheRead + usage.cacheWrite + usage.cacheWrite1h }), + }, + dumpTokenUsage: tokens, + }; +}; From 81ec8da8c2a1fb88f262f52de61611eca01f9580 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:02:42 +0800 Subject: [PATCH 131/156] fix(provider): preserve failures through state observation Publish access-only Codex terminal writes as deferred facts so a persistence error retains its original run outcome while the upstream401 remains readable. Observe Claude error JSON from a single byte read and relay those same bytes, preserving UTF8 BOM content without cloning the response. Actual provider runs cover full error bodies, original deferred rejection, and byte equality. --- .../__tests__/provider_test.ts | 9 +++++++ .../src/stages/observe-response.ts | 5 ++-- .../provider-codex/__tests__/fetch_test.ts | 2 +- .../provider-codex/__tests__/provider_test.ts | 27 +++++++++++++++++++ packages/provider-codex/src/backend.ts | 7 ----- .../src/stages/observe-response.ts | 1 + 6 files changed, 41 insertions(+), 10 deletions(-) diff --git a/packages/provider-claude-code/__tests__/provider_test.ts b/packages/provider-claude-code/__tests__/provider_test.ts index a989493256..fb44d7a94c 100644 --- a/packages/provider-claude-code/__tests__/provider_test.ts +++ b/packages/provider-claude-code/__tests__/provider_test.ts @@ -214,6 +214,15 @@ describe('createClaudeCodeProvider — Messages operation stages', () => { expect(bodies.every(body => JSON.stringify(body.messages[0]).includes('Shared system instructions'))).toBe(true); }); + test('observing a JSON error preserves its original UTF8 BOM bytes', async () => { + const body = { error: { type: 'permission_error', message: 'A beta feature is unavailable' } }; + const bytes = new TextEncoder().encode(`\uFEFF${JSON.stringify(body)}`); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(bytes, { status: 403, headers: { 'content-type': 'application/json', 'x-upstream': 'retained' } })); + const result = await collectChatProviderPipeline(createClaudeCodeProvider(currentRecord), 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [] }, undefined, noopAnthropicMessagesUpstreamCallOptions()); + expect(result.facts['response.claudeCode.failureBody']).toEqual(body); + expect(new Uint8Array(await result.response!.arrayBuffer())).toEqual(bytes); + }); + test('the Messages pipeline preserves a complete terminal refusal while recording state persistence', async () => { const body = { error: { type: 'permission_error', message: 'OAuth authentication is currently not allowed for this organization' }, diagnostic: 'x'.repeat(800) }; vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(body, { status: 403, headers: { 'x-upstream': 'retained' } })); diff --git a/packages/provider-claude-code/src/stages/observe-response.ts b/packages/provider-claude-code/src/stages/observe-response.ts index 4644820df7..7d117e6ea4 100644 --- a/packages/provider-claude-code/src/stages/observe-response.ts +++ b/packages/provider-claude-code/src/stages/observe-response.ts @@ -26,12 +26,13 @@ export const observeClaudeCodeResponse = (upstreamId: string) => defineStage {}); try { failureBody = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; failureBody = text; } const terminal = detectTerminalSentinel(response.status, text); if (terminal !== null) tasks.push(persistTerminalAccountState(upstreamId, terminal, response.status === 400 ? 'org_disabled_400_sentinel' : 'org_banned_403_sentinel', response.status, use.log.warn)); - retained = takeHttpResponse(new Response(text, { status: exchange.status, statusText: exchange.statusText, headers: response.headers })); + retained = takeHttpResponse(new Response(bytes, { status: exchange.status, statusText: exchange.statusText, headers: response.headers })); } return move({ ...back, 'response.http.exchange': retained, 'response.http.body': retained.body, diff --git a/packages/provider-codex/__tests__/fetch_test.ts b/packages/provider-codex/__tests__/fetch_test.ts index cdce55b656..285ea84d96 100644 --- a/packages/provider-codex/__tests__/fetch_test.ts +++ b/packages/provider-codex/__tests__/fetch_test.ts @@ -1190,7 +1190,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { }); const upstreamBody = { error: { code: 'token_invalidated', message: 're-import required' } }; const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(errorJson(401, upstreamBody, { 'x-upstream-marker': 'kept' })); - const persistTerminalState = vi.fn(async () => { throw new Error('state write failed'); }); + const persistTerminalState = vi.fn(async () => {}); const result = await callCodexOpenAIResponses({ upstreamId, account: accessOnlyAccount, model, body: { input: [], stream: true }, headers: new Headers(), diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index 5a11ac4074..69fe979640 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -93,6 +93,33 @@ const oauthTokenResponse = (overrides: Partial<{ access_token: string; refresh_t }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); describe('createCodexProvider', () => { + test('access-only401 remains readable while terminal persistence failure reaches the run outcome', async () => { + current = accessOnlyRecord(freshAccessToken); + const writeError = new Error('terminal state write failed'); + repo.saveState.mockRejectedValue(writeError); + const body = { error: { code: 'token_invalidated', message: 're-import required' }, diagnostic: 'full upstream details' }; + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(body, { status: 401, headers: { 'x-upstream': 'retained' } })); + const provider = createCodexProvider(current); + const events: Event[] = []; + const executed = await run(provider.pipelines.openaiResponses!, move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ id: 'gpt-5.4', endpoints: { openaiResponses: {} } })), + 'request.provider.payload': { input: [], stream: true }, + 'request.http.callId': 0, 'request.http.headers': [], + }), { httpCall: () => noopUpstreamCallOptions(), recordProtocolFrames: (frames: AsyncIterable) => frames, dump: (event: Event) => { events.push(event); } }); + const exchange = executed.facts['response.http.exchange']; + if (exchange.type === 'transportFailure') throw exchange.error; + const response = exchangeResponse(exchange); + expect(response.status).toBe(401); + expect(response.headers.get('x-upstream')).toBe('retained'); + expect(await response.json()).toEqual(body); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + await expect(executed.drain()).rejects.toBe(writeError); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(events.find(event => event.type === 'deferred.settled' && event.outcome.status === 'rejected')).toMatchObject({ + deferred: (executed.facts as typeof executed.facts & Record)['response.codex.background'], outcome: { status: 'rejected', reason: writeError }, + }); + }); + test.each(['openaiResponses', 'openaiResponsesCompact'] as const)('%s pipeline keeps parsed private wire content and actual endpoint observations', async operation => { const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => operation === 'openaiResponses' ? new Response(sseResponse().body, { headers: { [CODEX_RESPONSES_LITE_HEADER]: 'true' } }) diff --git a/packages/provider-codex/src/backend.ts b/packages/provider-codex/src/backend.ts index c7c01bae57..343a838b6d 100644 --- a/packages/provider-codex/src/backend.ts +++ b/packages/provider-codex/src/backend.ts @@ -461,13 +461,6 @@ export const decodeCodexUpstreamError = (rawText: string): CodexUpstreamError => export const classifyCodexUnauthorizedResponse = async (opts: CodexBackendCallBase, response: Response, parsed: CodexUpstreamError): Promise => { const { rawText, code, message } = parsed; if (opts.account.refresh_token === null) { - // Access-only credentials retain the authoritative upstream reply even when - // the terminal-state write fails. - try { - await opts.effects.persistTerminalState('session_terminated', message); - } catch { - // The upstream response remains authoritative. - } return new Response(rawText, { status: 401, statusText: response.statusText, headers: response.headers }); } if (code === 'token_invalidated') { diff --git a/packages/provider-codex/src/stages/observe-response.ts b/packages/provider-codex/src/stages/observe-response.ts index f0ee9d5063..cebb8f44f4 100644 --- a/packages/provider-codex/src/stages/observe-response.ts +++ b/packages/provider-codex/src/stages/observe-response.ts @@ -25,6 +25,7 @@ export const observeCodexResponse = (config: CodexPipelineConfig, operation: Cod const parsed = decodeCodexUpstreamError(await response.text()); if (exchange.body !== null) setRelease(exchange.body, async () => {}); failureBody = parsed.body; + if (call.account.refresh_token === null) pending.push(call.effects.persistTerminalState('session_terminated', parsed.message)); classified = await classifyCodexUnauthorizedResponse(call, response, parsed); } else { classified = response; From 34dd60522d99d886376cca4b7ad88c4acc235525 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:06:02 +0800 Subject: [PATCH 132/156] fix: resolve Alpha backends only when search dispatches Keep provider selection as a fixed lazy service and memoize the first selected candidate for the hosted turn. An unused declaration starts no selection promise, while actual child launches retain their eager dispatch ordering. Verified 259 hosted search tests, including an unused declaration whose selected upstream is outside the API key scope. --- .../hosted-tools/search-sub-request_test.ts | 4 ++-- .../chat/openai-responses/hosted-tools_test.ts | 15 +++++++++++++++ .../hosted-tools/search-sub-request/execute.ts | 2 +- .../hosted-tools/search-sub-request/services.ts | 2 +- .../openai-responses/hosted-tools/web-search.ts | 4 +++- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts index 36a9243d3e..0a7cb5fb8a 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts @@ -31,13 +31,13 @@ for (const failed of [false, true]) { const invocation = runWebSearchSubRequest(parent, request, { session: { pageCache: new Map(), getProvider: () => { throw new Error('local provider must not dispatch in alpha mode'); }, apiKeyId: apiKey.id }, alpha: { - sessionId: 'search-session', candidate: Promise.resolve(stubAlphaSearchCandidate(async body => { + sessionId: 'search-session', resolveCandidate: async () => stubAlphaSearchCandidate(async body => { dispatched = true; expect(body).toEqual({ id: 'search-session', commands: request.commands, settings: request.settings, input: request.input }); expect(Object.isFrozen(body.commands)).toBe(true); if (failed) throw fault; return new Response(JSON.stringify({ output: 'documentation' }), { headers: { 'content-type': 'application/json' } }); - })), + }), }, }); if (failed) await expect(invocation).rejects.toBe(fault); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts index 744391ed0f..ede3a2e247 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts @@ -6889,3 +6889,18 @@ test('helper echoes distinguish namespaced functions and rewrite hosted allowed_ assertEquals(tools?.[1].type, 'web_search'); assertEquals(findResponseCompleted(frames).response.tool_choice, choice); }); + +test('an unused hosted web-search declaration starts no Alpha selection work', async () => { + makeStubDeps(); + await getRepo().webSearchConfig.save({ + provider: 'tavily', tavily: { apiKey: 'test-key' }, microsoftWebIq: { apiKey: '' }, jina: { apiKey: '' }, + passthroughOpenAiSearch: { enabled: true, upstreamId: 'outside-key-scope', model: 'gpt-search' }, + } satisfies WebSearchConfig); + mockResolveAlpha.mockRejectedValue(new Error('Selected OpenAI search upstream is outside this API key scope')); + const invocation = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']) }); + const script = scriptedRun([messageTurn('no search requested', 0)]); + const result = await runHostedWebSearch(invocation, mockChatGatewayCtx(), script.run); + if (result.type !== 'events') throw new Error('Expected an ordinary model response'); + await collectFrames(result.events); + assertEquals(mockResolveAlpha.mock.calls.length, 0); +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts index b89a583a64..d6c2d22a1a 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts @@ -33,7 +33,7 @@ export const runWebSearchCall = defineStage< const dispatch: AlphaSearchDispatcher = async (body, _signal, headers) => { const alpha = use.webSearch.alpha; if (alpha === undefined) throw new Error('Alpha Search was dispatched without a selected provider'); - const candidate = await alpha.candidate; + const candidate = await alpha.resolveCandidate(); const pipeline = candidate.provider.pipelines.alphaSearch; if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no Alpha Search pipeline`); const selector = use.rememberCandidates([candidate])[0]; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts index 95ecb6f5ba..e25d169ded 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts @@ -4,7 +4,7 @@ import type { ModelCandidate } from '@floway-dev/provider'; export interface WebSearchRuntime { readonly session: Omit; - readonly alpha?: { readonly candidate: Promise; readonly sessionId: string }; + readonly alpha?: { readonly resolveCandidate: () => Promise; readonly sessionId: string }; } export interface SearchServices extends GatewayServices { readonly webSearch: WebSearchRuntime } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts index b8a13fdc50..f6bc849fe7 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts @@ -20,6 +20,7 @@ import { resolveConfiguredWebSearchProvider } from '../../../tools/web-search/pr import type { ConfiguredWebSearchProvider } from '../../../tools/web-search/types.ts'; import type { OpenAIResponsesFunctionTool, OpenAIResponsesFunctionToolCallItem, OpenAIResponsesHostedTool, OpenAIResponsesInputItem, OpenAIResponsesOutputWebSearchCall, OpenAIResponsesTool, OpenAIResponsesWebSearchAction } from '@floway-dev/protocols/openai-responses'; import { collectOpenAIResponsesToolEntries, createRandomOpenAIResponsesItemId, WEB_SEARCH_HOSTED_TYPE_NAMES } from '@floway-dev/protocols/openai-responses'; +import type { ModelCandidate } from '@floway-dev/provider'; import { providerModelOf } from '@floway-dev/provider'; // Runtime set derived from the canonical tuple declared next to @@ -602,11 +603,12 @@ export const webSearchHostedTool: HostedToolRegistration = async (invocation, ga apiKeyId: gatewayCtx.apiKeyId, ...(gatewayCtx.abortSignal !== undefined ? { signal: gatewayCtx.abortSignal } : {}), }; + let selectedAlphaCandidate: Promise | undefined; const webSearch: WebSearchRuntime = { session, ...(webSearchConfig.passthroughOpenAiSearch.enabled ? { alpha: { - candidate: resolveAlphaSearchCandidate({ + resolveCandidate: () => selectedAlphaCandidate ??= resolveAlphaSearchCandidate({ config: webSearchConfig.passthroughOpenAiSearch, upstreamIds: gatewayCtx.upstreamIds, scheduler: gatewayCtx.backgroundScheduler, From 543e0e38829df508f7ec2651b2b72fca918cf8ea Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:12:04 +0800 Subject: [PATCH 133/156] refactor: retire mutable interceptor composition Provider operations now compose immutable stages and the shared HTTP terminal. Remove the unused interceptor workspace package, dependency and configuration entries, and update the owning audit guide and comments to the current contracts. Verified 1083 affected tests, frozen offline lock consistency, agent inventory, verification parity and scoped lint. Full lint and typecheck expose parent-pending async logger and non-chat fixture migrations; those are resolved by the incoming parent branch. --- .../skills/audit-copilot-workarounds/SKILL.md | 25 +++--- AGENTS.md | 1 - eslint.config.ts | 1 - packages/gateway/package.json | 1 - .../src/data-plane/pipeline/attempt-body.ts | 21 +---- packages/interceptor/__tests__/index_test.ts | 87 ------------------- packages/interceptor/package.json | 15 ---- packages/interceptor/src/index.ts | 50 ----------- packages/interceptor/tsconfig.json | 4 - packages/interceptor/vitest.config.ts | 8 -- packages/platform/src/image-processor.ts | 7 +- .../provider-copilot/__tests__/auth_test.ts | 4 +- .../__tests__/provider_test.ts | 20 +---- .../compress-images_test.ts | 2 +- .../abort-on-tool-argument-whitespace_test.ts | 2 +- .../abort-on-tool-argument-whitespace_test.ts | 2 +- .../src/claude-code-metadata.ts | 4 +- .../request.ts | 8 +- .../request.ts | 2 +- .../request.ts | 4 +- .../context-window-error.ts | 2 +- .../reasoning-effort.ts | 2 +- pnpm-lock.yaml | 9 -- 23 files changed, 39 insertions(+), 242 deletions(-) delete mode 100644 packages/interceptor/__tests__/index_test.ts delete mode 100644 packages/interceptor/package.json delete mode 100644 packages/interceptor/src/index.ts delete mode 100644 packages/interceptor/tsconfig.json delete mode 100644 packages/interceptor/vitest.config.ts diff --git a/.agents/skills/audit-copilot-workarounds/SKILL.md b/.agents/skills/audit-copilot-workarounds/SKILL.md index 0cc2029cc7..4c0fa17549 100644 --- a/.agents/skills/audit-copilot-workarounds/SKILL.md +++ b/.agents/skills/audit-copilot-workarounds/SKILL.md @@ -16,23 +16,24 @@ Copilot upstream. The provider code and the reference URLs beside each workaround are the inventory; there is no separate documentation list to reconcile. -1. Start from - `packages/provider-copilot/src/interceptors/{openai-chat-completions,anthropic-messages,openai-responses}/index.ts` - and `packages/provider-copilot/src/defaults.ts`. Follow every registered - interceptor and default-enabled shim to its implementation and tests. +1. Start from `packages/provider-copilot/src/pipelines.ts`, + `chat-pipelines.ts`, and `defaults.ts`. Follow every registered stage and + default-enabled shim to its implementation under `src/stages/` and its + mirrored tests under `__tests__/stages/`. 2. Sweep the rest of `packages/provider-copilot/src` for non-pricing reference URLs and for vendor constants, thresholds, timeouts, retries, and pinned wire values that require a reference but may be missing one. Pricing citations belong to `fetching-models-pricing`; everything else remains in this audit. -3. Include provider-level request/result shaping and catalog shaping even when - they are not interceptors. In particular, inspect `provider.ts`, +3. Include request/result shaping and catalog shaping. Inspect `provider.ts`, `fetch-models.ts`, `known-models.ts`, `model-selection.ts`, and - `merge-claude-variants.ts` together with their imports and tests. -4. Include the authentication fingerprint and management/data-plane behavior in - `auth.ts`, plus OpenAI Responses item identity and replay handling rooted at - `interceptors/openai-responses/item-id-membrane.ts`. Follow adjacent carrier and - compaction modules rather than assuming the interceptor registry contains the - whole workaround. + `merge-claude-variants.ts` together with their imports and tests. Exercise + data-plane behavior through the provider's operation pipelines rather than + its catalog-only instance. +4. Include the authentication fingerprint and control/data-plane behavior in + `auth.ts` and `control-http.ts`, plus OpenAI Responses item identity and replay + handling rooted at `stages/openai-responses/item-id-membrane.ts` and + `opaque-item-id.ts`. Follow adjacent carrier and compaction modules; the + operation assemblies are entrypoints, not a complete workaround inventory. 5. Record each item's owning module, reference URLs, affected source and target APIs, models, account scope, default flag state, tests, and exit condition: delete an obsolete workaround, refresh pinned mimicry, or retain it with diff --git a/AGENTS.md b/AGENTS.md index 48921f82d8..2561236ab3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,6 @@ | Package | `packages/agent-setup` | Configures supported coding agents. | | Package | `packages/gateway` | Composes gateway services. | | Package | `packages/http` | Provides HTTP transport primitives. | -| Package | `packages/interceptor` | Chains provider boundary rewrites. | | Package | `packages/pipeline` | Runs stages over a fact record. | | Package | `packages/platform` | Defines portable runtime contracts. | | Package | `packages/protocols` | Defines protocol contracts. | diff --git a/eslint.config.ts b/eslint.config.ts index 91dcee2f72..72af327ee1 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -71,7 +71,6 @@ const projectList = [ './packages/agent-setup/tsconfig.scripts.json', './packages/gateway/tsconfig.json', './packages/http/tsconfig.json', - './packages/interceptor/tsconfig.json', './packages/pipeline/tsconfig.json', './packages/platform/tsconfig.json', './packages/protocols/tsconfig.json', diff --git a/packages/gateway/package.json b/packages/gateway/package.json index e8975fc677..4620164867 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -38,7 +38,6 @@ "dependencies": { "@floway-dev/agent-setup": "workspace:*", "@floway-dev/http": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/pipeline": "workspace:*", "@floway-dev/platform": "workspace:*", "@floway-dev/protocols": "workspace:*", diff --git a/packages/gateway/src/data-plane/pipeline/attempt-body.ts b/packages/gateway/src/data-plane/pipeline/attempt-body.ts index e84c7a300c..34075be29a 100644 --- a/packages/gateway/src/data-plane/pipeline/attempt-body.ts +++ b/packages/gateway/src/data-plane/pipeline/attempt-body.ts @@ -1,21 +1,6 @@ -// What a provider is handed when a stage dials it. -// -// The record's own children travel, by identity. One object is built here, and only because -// this is where the addressed model is stamped and then dropped; everything under it is the -// record's, frozen — a fact that could be edited after the event is not a fact — and a provider -// shapes the body it sends by rebuilding rather than by writing into what it was given. So both -// sides of this boundary agree and there is nothing to defend against. -// -// It was not always so, and the shape of the failure is worth keeping: three Copilot rules -// wrote one level down, into a message or a content block, which against a frozen record threw -// `Cannot add property …, object is not extensible` — as a 502, from the point in the stack -// least able to explain it. A deep copy here absorbed that, at the cost of one pass over every -// payload on every dial, and at the larger cost of making immutability something the gateway -// worked around. The three rules rebuild now, and so does every alias overlay: an overlay that -// wrote in place would be safe only for as long as this function happened to have rebuilt the -// level it writes to, which is the same trap one level up. -// -// So what is left is only the three things every wire was otherwise writing by hand. +// Alias rules apply to the addressed candidate. Providers receive model-free +// request content and stamp their own wire model; unchanged descendants retain +// their identity through the immutable handoff. import type { ModelCandidate } from '@floway-dev/provider'; diff --git a/packages/interceptor/__tests__/index_test.ts b/packages/interceptor/__tests__/index_test.ts deleted file mode 100644 index 3453838fec..0000000000 --- a/packages/interceptor/__tests__/index_test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { test } from 'vitest'; - -import { type Interceptor, runInterceptors } from '../src/index.ts'; -import { assertEquals, assertRejects } from '@floway-dev/test-utils'; - -type TestCtx = { payload: { value: string } }; - -test('composes interceptors outermost-first and unwinds epilogues inside-out', async () => { - const calls: string[] = []; - - const outer: Interceptor = async (_ctx, run) => { - calls.push('outer-before'); - const result = await run(); - calls.push('outer-after'); - return result; - }; - const inner: Interceptor = async (_ctx, run) => { - calls.push('inner-before'); - const result = await run(); - calls.push('inner-after'); - return result; - }; - - await runInterceptors({ payload: { value: 'ok' } }, [outer, inner], () => { - calls.push('terminal'); - return Promise.resolve('done'); - }); - - assertEquals(calls, ['outer-before', 'inner-before', 'terminal', 'inner-after', 'outer-after']); -}); - -test('lets an interceptor retry by calling run() again — each call reruns the inner chain', async () => { - const ctx: TestCtx = { payload: { value: 'broken' } }; - let attempts = 0; - - const interceptor: Interceptor = async (current, run) => { - const first = await run(); - if (first !== 'fail') return first; - current.payload.value = 'fixed'; - return await run(); - }; - - const result = await runInterceptors(ctx, [interceptor], () => { - attempts += 1; - return Promise.resolve(ctx.payload.value === 'broken' ? 'fail' : ctx.payload.value); - }); - - assertEquals(attempts, 2); - assertEquals(result, 'fixed'); -}); - -test('propagates an inner throw past each enclosing run() call site without swallowing', async () => { - const seen: string[] = []; - - const wrap = (label: string): Interceptor => async (_ctx, run) => { - seen.push(`${label}-before`); - try { - return await run(); - } finally { - seen.push(`${label}-after`); - } - }; - - const boom = new Error('upstream blew up'); - await assertRejects( - () => runInterceptors({ payload: { value: 'x' } }, [wrap('outer'), wrap('inner')], () => Promise.reject(boom)), - Error, - 'upstream blew up', - ); - - assertEquals(seen, ['outer-before', 'inner-before', 'inner-after', 'outer-after']); -}); - -test('lets an interceptor patch context before run and transform the result after run', async () => { - const ctx: TestCtx = { payload: { value: 'original' } }; - - const interceptor: Interceptor = async (current, run) => { - current.payload.value = 'patched'; - const result = await run(); - return `${result}:${current.payload.value}`; - }; - - const result = await runInterceptors(ctx, [interceptor], () => Promise.resolve(ctx.payload.value)); - - assertEquals(ctx.payload.value, 'patched'); - assertEquals(result, 'patched:patched'); -}); diff --git a/packages/interceptor/package.json b/packages/interceptor/package.json deleted file mode 100644 index ba6e7f7397..0000000000 --- a/packages/interceptor/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "@floway-dev/interceptor", - "private": true, - "version": "0.0.0", - "type": "module", - "exports": { - ".": { "import": "./src/index.ts", "types": "./src/index.ts" } - }, - "scripts": { - "typecheck": "tsc --noEmit" - }, - "devDependencies": { - "@floway-dev/test-utils": "workspace:*" - } -} diff --git a/packages/interceptor/src/index.ts b/packages/interceptor/src/index.ts deleted file mode 100644 index fe156d1794..0000000000 --- a/packages/interceptor/src/index.ts +++ /dev/null @@ -1,50 +0,0 @@ -// Interceptors wrap a single typed call. Each interceptor receives the call's -// own invocation state and a `run` to delegate to the next interceptor (the -// innermost run executes the call itself). Interceptors may inspect or mutate -// that invocation state before `run`, await `run` and transform the result, -// short-circuit by returning without calling `run`, or retry by invoking `run` -// again. The shape is generic in Ctx/Result so it works for any kind of call, -// wired by the caller into concrete chains. -// -// `Ctx` carries the call itself — payload, headers, chosen target — and is the -// slot interceptors write to. There is no ambient second slot: every chain this -// runs is a provider boundary chain, and a boundary has nothing around the call -// to hand down. -// -// ## Mutation convention -// -// Mutations applied to `ctx` before `run()` propagate forward through every -// downstream interceptor and into the terminal call. They are **one-way**: the -// interceptor that wrote a field does not restore it on the way out, and the -// framework does not snapshot/rewind state for it. Whatever consumes the -// chain's output post-run (the caller that invoked `runInterceptors`, an outer -// interceptor's after-`run()` code) must keep its own captured copy of any input -// it still needs. -// -// This is a provider-boundary convention and not the gateway's: what a request -// travels in through the gateway is a fact record, which is frozen at every -// handover and rebuilt rather than written to. The two live side by side because -// they answer different questions — a boundary shapes one wire call it owns -// outright, a pipeline carries a turn nobody owns alone. -// -// The convention exists because partial adoption is the worst case: if some -// interceptors restore and others don't, there is no honest invariant the -// rest of the codebase can rely on — readers can no longer tell what `ctx` -// will look like at any given seam without auditing every interceptor in the -// chain. Forbidding restore everywhere is the only way to get a single -// predictable shape. -// -// The framework does not enforce this; reviewers do. A new interceptor that -// writes `ctx.foo = bar` in `try` and `ctx.foo = original` in `finally` is a -// convention violation, not a feature. -export type InterceptorRun = () => Promise; -export type Interceptor = (ctx: Ctx, run: InterceptorRun) => Promise; - -export const runInterceptors = async ( - ctx: Ctx, - interceptors: readonly Interceptor[], - terminal: InterceptorRun, -): Promise => { - const run = (index: number): Promise => (index < interceptors.length ? interceptors[index](ctx, () => run(index + 1)) : terminal()); - return await run(0); -}; diff --git a/packages/interceptor/tsconfig.json b/packages/interceptor/tsconfig.json deleted file mode 100644 index 4cffc03465..0000000000 --- a/packages/interceptor/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "include": ["vitest.config.ts", "src/**/*.ts", "__tests__/**/*.ts"] -} diff --git a/packages/interceptor/vitest.config.ts b/packages/interceptor/vitest.config.ts deleted file mode 100644 index a94d8e60e5..0000000000 --- a/packages/interceptor/vitest.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from 'vitest/config'; - -export default defineConfig({ - test: { - include: ['__tests__/**/*_test.ts'], - environment: 'node', - }, -}); diff --git a/packages/platform/src/image-processor.ts b/packages/platform/src/image-processor.ts index 03569b5f62..8488ac8274 100644 --- a/packages/platform/src/image-processor.ts +++ b/packages/platform/src/image-processor.ts @@ -91,10 +91,9 @@ export const compressBytesToWebp = async ( // In-memory passthrough used by tests. There is no WebP codec available under // the test runtime, so this stub returns the input bytes unchanged; it exists -// only to satisfy the ImageProcessor contract so the egress interceptors run -// end-to-end. Interceptor behaviour (which images are rewritten, what target -// is computed) is asserted against dedicated spy processors in the -// interceptor tests, not against this stub. +// only to satisfy the ImageProcessor contract so the provider stages run +// end-to-end. Image selection and target dimensions are asserted against +// dedicated spy processors in the owning stage tests. export const createInMemoryImageProcessor = (): ImageProcessor => ({ compressToWebp: input => Promise.resolve(input), }); diff --git a/packages/provider-copilot/__tests__/auth_test.ts b/packages/provider-copilot/__tests__/auth_test.ts index f0fce4bf6e..8ae3d02f1d 100644 --- a/packages/provider-copilot/__tests__/auth_test.ts +++ b/packages/provider-copilot/__tests__/auth_test.ts @@ -251,7 +251,7 @@ describe('Copilot token exchange retries', () => { }); }); -test('copilotAuthedFetch overlays interceptor headers on the pinned base set', async () => { +test('copilotAuthedFetch overlays request headers on the pinned base set', async () => { await mockTokenAndCapture([['x-initiator', 'agent'], ['copilot-vision-request', 'true']], headers => { assertEquals(headers.get('x-initiator'), 'agent'); assertEquals(headers.get('copilot-vision-request'), 'true'); @@ -261,7 +261,7 @@ test('copilotAuthedFetch overlays interceptor headers on the pinned base set', a }); }); -test('copilotAuthedFetch deletes a base header when the interceptor passes an empty-string value', async () => { +test('copilotAuthedFetch deletes a base header when the caller passes an empty-string value', async () => { // Sentinel contract: empty string means drop this base header from the pinned set. await mockTokenAndCapture([['copilot-integration-id', '']], headers => { assertEquals(headers.has('copilot-integration-id'), false); diff --git a/packages/provider-copilot/__tests__/provider_test.ts b/packages/provider-copilot/__tests__/provider_test.ts index 08aa682f84..477079b0ba 100644 --- a/packages/provider-copilot/__tests__/provider_test.ts +++ b/packages/provider-copilot/__tests__/provider_test.ts @@ -611,8 +611,6 @@ test('Copilot provider sets copilot-vision-request when an image is nested insid const provider = instance.instance; const visionHeaders: (string | null)[] = []; - // The vision-detection interceptor runs inside `provider.callAnthropicMessages`, so - // it must walk into nested `tool_result.content` to find the image. const driveAnthropicMessages = async (providerModel: Awaited>[number], body: Omit): Promise => { await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, body, undefined, noopAnthropicMessagesUpstreamCallOptions()); }; @@ -689,12 +687,8 @@ test('Copilot provider sets copilot-vision-request when an image is nested insid }); test('Copilot Anthropic Messages boundary chain does NOT fire on the OpenAI Chat Completions wire (translated path)', async () => { - // Boundary isolation: each provider call method runs only its own protocol - // boundary chain. The Anthropic-Messages-only `withClaudeAgentHeadersSet` interceptor - // would set x-interaction-type to 'messages-proxy' for Claude Code SDK - // metadata, but it MUST NOT run when the translated path calls Copilot's - // openai-chat-completions wire — that path runs `COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY`, - // which has no Anthropic-Messages-source headers in it. + // The operation chain selects wire-specific stages. Claude Code metadata must + // not install Anthropic Messages headers on a Chat Completions request. const { copilotUpstream } = await setupCopilotTest(); const instance = createCopilotProvider(copilotUpstream); const provider = instance.instance; @@ -719,9 +713,6 @@ test('Copilot Anthropic Messages boundary chain does NOT fire on the OpenAI Chat }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - // Even with a Claude-Code-shaped metadata blob, the openai-chat-completions - // boundary chain has no Anthropic-Messages-source interceptor, so the - // messages-proxy intent must not appear on the wire. await collectChatProviderPipeline(instance, 'openaiChatCompletions', providerModel, { messages: [{ role: 'user', content: 'hi' }], metadata: { user_id: JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }) }, @@ -729,11 +720,8 @@ test('Copilot Anthropic Messages boundary chain does NOT fire on the OpenAI Chat }, ); - // The openai-chat-completions wire defaults to `conversation-agent` (set by - // `copilotAuthedFetch` in `packages/provider-copilot/src/auth.ts`). The - // Anthropic-Messages-boundary `withClaudeAgentHeadersSet` would overwrite it to - // `messages-proxy` if it had run — its absence is the - // proof that the Anthropic Messages boundary chain did NOT fire on this wire. + // Authentication sets conversation-agent; the Anthropic-only header stage + // would overwrite it to messages-proxy if it ran on this operation. assertEquals(observedInteractionType, ['conversation-agent']); }); diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts index 88d77fa908..9a9749ee7a 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts @@ -195,7 +195,7 @@ test('high-res Opus clamps the long edge to 2576 on very wide images', async () assertEquals(await capTargetFor('claude-opus-4.7', { width: 8000, height: 1000 }), { width: 2576, height: 322 }); }); -// Drives the interceptor end-to-end with a header-only PNG of the requested +// Drives the stage end-to-end with a header-only PNG of the requested // dimensions and reads back the target the per-model cap resolved to. The // PNG carries no pixel payload — image-size reads dimensions from IHDR and // does not validate the rest of the file. diff --git a/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts index 2c6affa59c..f7b3a3c7e5 100644 --- a/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts @@ -51,7 +51,7 @@ const runExpectingThrow = async (frames: ProtocolFrame { diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts index 8123bdce89..fcc28be1da 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts @@ -72,7 +72,7 @@ test('aborts and emits an error event + done when whitespace exceeds the thresho const wsDelta = '\n'.repeat(MAX_CONSECUTIVE_WHITESPACE + 1); const frames: ProtocolFrame[] = [ eventFrame(argsDelta(0, wsDelta)), - // Should not be observed: interceptor aborts on the first offending delta. + // The stage aborts before the second delta can reach the client. eventFrame(argsDelta(0, '\n\n\n')), doneFrame(), ]; diff --git a/packages/provider-copilot/src/claude-code-metadata.ts b/packages/provider-copilot/src/claude-code-metadata.ts index 4762a6217c..3fb6f74e8b 100644 --- a/packages/provider-copilot/src/claude-code-metadata.ts +++ b/packages/provider-copilot/src/claude-code-metadata.ts @@ -11,8 +11,8 @@ * attempt JSON parsing — Claude Code never combines the two formats, and * skipping the parse avoids a thrown `SyntaxError` allocation on the hot path. * - * Used by withClaudeAgentHeadersSet (needs BOTH halves to fire) and by - * withInteractionIdHeaderSet (needs only sessionId). + * The Claude agent header stage needs BOTH halves to fire; the interaction ID + * stage needs only sessionId. * * References: * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/utils.ts#L151 diff --git a/packages/translate/src/anthropic-messages-via-openai-chat-completions/request.ts b/packages/translate/src/anthropic-messages-via-openai-chat-completions/request.ts index afc6efcce3..2761b585a6 100644 --- a/packages/translate/src/anthropic-messages-via-openai-chat-completions/request.ts +++ b/packages/translate/src/anthropic-messages-via-openai-chat-completions/request.ts @@ -196,9 +196,9 @@ const translateAnthropicMessagesAssistant = (message: AnthropicMessagesAssistant }; // Anthropic Messages system blocks are prompt boundaries; preserve each one -// as a separate OpenAI Chat Completions text part so a CC→Anthropic Messages→CC round trip -// does not silently merge them. Falls back to the simple string form when -// the source is already a single-string field. +// as a separate OpenAI Chat Completions text part so an OpenAI Chat Completions → +// Anthropic Messages → OpenAI Chat Completions round trip does not merge them. +// Preserve the simple string form when the source is already a string. const systemContentFromBlocks = (system: string | AnthropicMessagesTextBlock[]): string | OpenAIChatCompletionsContentPart[] => typeof system === 'string' ? system @@ -266,7 +266,7 @@ const translateAnthropicMessagesToolChoice = (toolChoice?: AnthropicMessagesPayl export const buildTargetRequest = (payload: AnthropicMessagesPayload): OpenAIChatCompletionsPayload => { const clientTools = filterAnthropicMessagesClientTools(payload.tools); // Pass effort through verbatim; per-upstream enum acceptance (e.g. some - // backends rejecting `xhigh`/`max`) is the target interceptor's concern. + // backends rejecting `xhigh`/`max`) is the target provider's concern. const reasoningEffort = resolveAnthropicMessagesReasoningEffort(payload); const jsonSchema = openAiJsonSchemaCoreFromAnthropicMessagesFormat(payload.output_config?.format); const responseFormat = jsonSchema ? { type: 'json_schema' as const, json_schema: jsonSchema } : undefined; diff --git a/packages/translate/src/openai-chat-completions-via-anthropic-messages/request.ts b/packages/translate/src/openai-chat-completions-via-anthropic-messages/request.ts index 04135213fe..3089332dc8 100644 --- a/packages/translate/src/openai-chat-completions-via-anthropic-messages/request.ts +++ b/packages/translate/src/openai-chat-completions-via-anthropic-messages/request.ts @@ -153,7 +153,7 @@ const buildAnthropicMessagesInput = async (messages: OpenAIChatCompletionsMessag // was hoisted earlier). Anthropic upstreams diverge on inline // role:'system' here (Bedrock accepts it under placement rules; // Vertex rejects it outright), so the gateway's - // `rewrite-mid-conv-system-to-user` interceptor flag is the safety + // `rewrite-mid-conv-system-to-user` role-compatibility flag is the safety // net for any inline system that would otherwise reach an upstream // that does not accept it. const blocks = convertSystemContent(message.content); diff --git a/packages/translate/src/openai-responses-via-openai-chat-completions/request.ts b/packages/translate/src/openai-responses-via-openai-chat-completions/request.ts index 8bfe721fbe..0b05aede74 100644 --- a/packages/translate/src/openai-responses-via-openai-chat-completions/request.ts +++ b/packages/translate/src/openai-responses-via-openai-chat-completions/request.ts @@ -88,8 +88,8 @@ const translateOpenAIResponsesTools = (tools: OpenAIResponsesTool[] | null | und // After allowed_tools selection, Chat Completions can represent only flat // function and custom declarations. Custom tools are wrapped as functions // and recorded so response events can restore their freeform shape. The - // server-tool shim rewrites hosted web_search declarations to ordinary - // function tools before this translation, so retained shim tools need no + // hosted-tools stages rewrite web_search declarations to ordinary + // function tools before this translation, so retained hosted tools need no // special handling here. const out: OpenAIChatCompletionsTool[] = []; diff --git a/packages/translate/src/shared/anthropic-messages-via/context-window-error.ts b/packages/translate/src/shared/anthropic-messages-via/context-window-error.ts index a873b62e51..01f30158d3 100644 --- a/packages/translate/src/shared/anthropic-messages-via/context-window-error.ts +++ b/packages/translate/src/shared/anthropic-messages-via/context-window-error.ts @@ -19,7 +19,7 @@ import { buildPromptTooLongBody } from '@floway-dev/protocols/anthropic-messages // - Copilot `/v1/messages` (Anthropic-shaped body carrying a Copilot-specific // message string): // {"type":"error","error":{"type":"invalid_request_error","message":"Request body is too large for model context window ..."}} -// The provider-copilot Anthropic Messages interceptor handles this at the provider +// The provider-copilot Anthropic Messages error-rewrite stage handles this at the provider // layer; the substring is listed here so an Anthropic-Messages-via-OpenAI-Responses trip that // ever encounters the same phrasing behaves consistently. diff --git a/packages/translate/src/shared/anthropic-messages-via/reasoning-effort.ts b/packages/translate/src/shared/anthropic-messages-via/reasoning-effort.ts index 17ef179645..e101588815 100644 --- a/packages/translate/src/shared/anthropic-messages-via/reasoning-effort.ts +++ b/packages/translate/src/shared/anthropic-messages-via/reasoning-effort.ts @@ -6,7 +6,7 @@ import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-m // // 1. `output_config.effort` — gateway-canonical OpenAI-style override; passes // through verbatim. Per-upstream enum acceptance is the target -// interceptor's concern. +// provider's concern. // 2. `thinking.type === 'disabled'` — emit the `'none'` sentinel. // 3. `thinking.type === 'enabled'` / `'adaptive'` — caller asked for thinking // but did not pin an effort. Emit `'medium'` (the OpenAI/Azure default diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 39285d7a0e..1b3f4ffd4f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -333,9 +333,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline @@ -428,12 +425,6 @@ importers: specifier: ^2.2.0 version: 2.2.0 - packages/interceptor: - devDependencies: - '@floway-dev/test-utils': - specifier: workspace:* - version: link:../test-utils - packages/pipeline: dependencies: '@noble/hashes': From 4c7caf5b3d59102e6b56bda42c6f9dee5e21ccc0 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:14:11 +0800 Subject: [PATCH 134/156] fix: retain accepted counting and compaction statuses --- .../chat/anthropic-messages-count-tokens_test.ts | 7 +++++++ .../chat/gemini-generate-content-count-tokens_test.ts | 7 +++++++ .../chat/anthropic-messages/count-tokens/emit.ts | 2 +- .../chat/gemini-generate-content/count-tokens/emit.ts | 2 +- .../src/data-plane/chat/openai-responses/compact/emit.ts | 2 +- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts index 1f43cffba5..05ae302063 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts @@ -338,3 +338,10 @@ describe('the messages count-tokens chain', () => { expect((facts as Record)['response.usage.billable']).toEqual([]); }); }); + +it('preserves the accepted native counting HTTP status', async () => { + resolves([candidate(async () => ({ response: new Response(JSON.stringify({ input_tokens: 3 }), { status: 201, headers: { 'content-type': 'application/json' } }), modelKey: 'counted-model' }))]); + const { facts, drain } = await count(); + expect(facts['response.http.status']).toBe(201); + await drain(); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts index ecc2ed3ce8..737e962b60 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts @@ -228,3 +228,10 @@ describe('the Gemini generateContent count-tokens chain', () => { expect((facts as Record)['response.usage.billable']).toEqual([]); }); }); + +it('preserves the accepted native counting HTTP status', async () => { + resolves([candidate(async () => ({ response: new Response(JSON.stringify({ input_tokens: 3 }), { status: 201, headers: { 'content-type': 'application/json' } }), modelKey: 'counted-model' }))]); + const { facts, drain } = await count(); + expect(facts['response.http.status']).toBe(201); + await drain(); +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts index 2701fa356e..f96f4c7cdb 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts @@ -48,7 +48,7 @@ export const emitAnthropicMessagesTokenCount = defineStage< ...rest, 'response.http.headers': forClient, 'response.chat.anthropicMessages.rendered': move(answer.body as Record), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; }, }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts index 3cbafa02bc..8abe520534 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts @@ -48,7 +48,7 @@ export const emitGeminiGenerateContentTokenCount = defineStage< ...rest, 'response.http.headers': forClient, 'response.chat.geminiGenerateContent.rendered': move(answer.body as Record), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts index 13074d0e7f..0d4a6adc2a 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts @@ -81,7 +81,7 @@ export const emitOpenAIResponsesCompaction = defineStage< 'response.chat.openaiResponses.rendered': move( completeOpenAIResponsesCompaction(persisted, openaiResponsesCreatedAt(use.gateway)) as unknown as Record, ), - 'response.http.status': 200, + 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, 'response.chat.openaiResponses.streamedUsage': move( withVerdict(rest['response.chat.openaiResponses.streamedUsage'], compactionFailed(persisted)), ), From f5eb3c77354e9d8559cb564b8875d4251b44f27a Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:32:59 +0800 Subject: [PATCH 135/156] Complete recorded source after nonstream protocol collection Keep terminal-based collectors from closing their recorded cursor prematurely, then exhaust that same cursor to publish a real stream.end. Verify completed HTTP generation and compaction dumps retain source-exhaustion proof. --- .../data-plane/chat/run-dump-frames_test.ts | 13 ++++++++++++- .../src/data-plane/chat/anthropic-messages/emit.ts | 4 ++-- .../data-plane/chat/gemini-generate-content/emit.ts | 4 ++-- .../data-plane/chat/openai-chat-completions/emit.ts | 4 ++-- .../chat/openai-responses/compact/emit.ts | 3 ++- .../src/data-plane/chat/openai-responses/emit.ts | 4 ++-- .../src/data-plane/chat/shared/client-stream.ts | 12 ++++++++++++ 7 files changed, 34 insertions(+), 10 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts index 71dd9353ed..ca2d4b0fe6 100644 --- a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts @@ -16,6 +16,7 @@ import { test, vi } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; import { copilotModels, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseOpenAIResponsesResponse } from '../../test-utils/app.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; /** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, @@ -78,7 +79,17 @@ const framesRecordedBy = async (turn: (apiKey: string) => Promise): Pr await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); const stored = dumps.stored[0]; assertExists(stored); - return resolveFrames(eventsOf(runRecordOf(stored.record))); + const events = eventsOf(runRecordOf(stored.record)); + const read = createRunReader(); + let clientStream: number | undefined; + for (const event of events) { + const decoded = read(event as unknown as DumpEvent); + const frames = decoded?.facts?.['response.chat.clientFrames']; + if (typeof frames === 'object' && frames !== null && 'stream' in frames) clientStream = frames.stream as number; + } + assertExists(clientStream); + assertEquals(events.some(event => event.type === 'stream.end' && event.streamId === clientStream), true); + return resolveFrames(events); }; /** The content is the point, not the count: a record holding frames with nothing in them is diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts index e66b5d1edc..9455c1064c 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts @@ -6,7 +6,7 @@ import { isFailure, renderFailure, mintedAs } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { bindClientRelease, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { bindClientRelease, collectClientFrames, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { collectAnthropicMessagesProtocolEventsToResult, anthropicMessagesEventToSsePayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; @@ -82,7 +82,7 @@ export const emitAnthropicMessages = defineStage< 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, 'response.chat.anthropicMessages.rendered': move( - await collectAnthropicMessagesProtocolEventsToResult(frames) as unknown as Record, + await collectClientFrames(frames, collectAnthropicMessagesProtocolEventsToResult) as unknown as Record, ), 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts index 3efe0bfb2d..73b0253f77 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts @@ -6,7 +6,7 @@ import { isFailure, renderFailure } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { bindClientRelease, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { bindClientRelease, collectClientFrames, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectGeminiGenerateContentProtocolEventsToResult, type GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; @@ -85,7 +85,7 @@ export const emitGeminiGenerateContent = defineStage< 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, 'response.chat.geminiGenerateContent.rendered': move( - await collectGeminiGenerateContentProtocolEventsToResult(frames) as unknown as Record, + await collectClientFrames(frames, collectGeminiGenerateContentProtocolEventsToResult) as unknown as Record, ), 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts index 1511948f88..0ecdbc3c96 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts @@ -5,7 +5,7 @@ import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/fa import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { bindClientRelease, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { bindClientRelease, collectClientFrames, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { eventFrame, isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectOpenAIChatCompletionsProtocolEventsToResult, openaiChatCompletionsProtocolFrameToSSEFrame, type OpenAIChatCompletionsStreamEvent, type ClientOpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; @@ -84,7 +84,7 @@ export const emitOpenAIChatCompletions = defineStage< 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, 'response.chat.openaiChatCompletions.rendered': move( - await collectOpenAIChatCompletionsProtocolEventsToResult(frames) as unknown as Record, + await collectClientFrames(frames, collectOpenAIChatCompletionsProtocolEventsToResult) as unknown as Record, ), 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts index 0d4a6adc2a..5a05329329 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts @@ -5,6 +5,7 @@ import { isFailure, renderFailure, mintedErrorEnvelope } from '../../../pipeline import type { StreamOutcome } from '../../../pipeline/serve.ts'; import { isForwardableUpstreamHeader } from '../../../shared/upstream-response.ts'; import type { ChatServices } from '../../services.ts'; +import { collectClientFrames } from '../../shared/client-stream.ts'; import { wrapOpenAIResponsesStatefulOutput, openaiResponsesCreatedAt } from '../client-output.ts'; import { completeOpenAIResponsesCompaction } from '../compaction-resource.ts'; import { internalErrorEnvelope } from '../errors.ts'; @@ -73,7 +74,7 @@ export const emitOpenAIResponsesCompaction = defineStage< wrapOpenAIResponsesStatefulOutput(answer.frames as AsyncIterable>, use.gateway), use.gateway.dump, ); - const persisted = await collectOpenAIResponsesProtocolEventsToResult(frames); + const persisted = await collectClientFrames(frames, collectOpenAIResponsesProtocolEventsToResult); return { ...rest, 'response.chat.clientFrames': move(frames), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts index 7ea2fc7716..7f18afa066 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts @@ -5,7 +5,7 @@ import { recordStream } from '../../../dump/run-sink.ts'; import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; import type { ChatServices } from '../services.ts'; -import { bindClientRelease, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { bindClientRelease, collectClientFrames, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { collectOpenAIResponsesProtocolEventsToResult, openaiResponsesProtocolFrameToSSEFrame, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent, type ClientOpenAIResponsesStreamEvent, type ClientResponseResource } from '@floway-dev/protocols/openai-responses'; @@ -105,7 +105,7 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move( - await collectOpenAIResponsesProtocolEventsToResult(frames) as unknown as Record, + await collectClientFrames(frames, collectOpenAIResponsesProtocolEventsToResult) as unknown as Record, ), 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, }; diff --git a/packages/gateway/src/data-plane/chat/shared/client-stream.ts b/packages/gateway/src/data-plane/chat/shared/client-stream.ts index 96ef8fe711..d660047b36 100644 --- a/packages/gateway/src/data-plane/chat/shared/client-stream.ts +++ b/packages/gateway/src/data-plane/chat/shared/client-stream.ts @@ -3,6 +3,18 @@ import type { StreamOutcome } from '../../pipeline/serve.ts'; import { defer, setRelease, type Owned, type Deferred } from '@floway-dev/pipeline'; import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; +// Collectors stop at the protocol terminal. Keep that stop local to the fold so the same +// recorded cursor can still prove source exhaustion and publish its stream.end marker. +export const collectClientFrames = async ( + frames: AsyncIterable, + collect: (frames: AsyncIterable) => Promise, +): Promise => { + const cursor = frames[Symbol.asyncIterator](); + const result = await collect({ [Symbol.asyncIterator]: () => ({ next: () => cursor.next() }) }); + while (!(await cursor.next()).done) continue; + return result; +}; + export const framedClientStream = >( source: AsyncIterable, render: (frame: Frame) => SseFrame, From 3af29bac241204ecd9f7a99defab174a704499c4 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:36:29 +0800 Subject: [PATCH 136/156] refactor: make run dumps use streaming storage exclusively Remove the duplicate buffered production write surface now that every run uses putRun. Prepared fixtures exercise the actual streaming contract through test-only support; detail reads still return the complete stored artifact. Updated storage and frame documentation. Verified 62 affected tests, gateway typecheck and scoped lint, preserving retention, unique-file collision handling, staged leases and bytes-before-metadata publication. --- README.md | 6 +- .../gateway/__tests__/dump/test-fixtures.ts | 12 ++-- packages/gateway/__tests__/dump/write-run.ts | 9 +++ .../gateway/__tests__/repo/dump-store_test.ts | 45 ++++++------- .../scheduled/expiration-sweeps_test.ts | 13 ++-- packages/gateway/__tests__/scheduled_test.ts | 7 ++- packages/gateway/__tests__/vitest.setup.ts | 1 - packages/gateway/src/dump/store-contract.ts | 8 +-- packages/gateway/src/dump/types.ts | 29 +++------ packages/gateway/src/repo/dump-store.ts | 63 ++----------------- 10 files changed, 66 insertions(+), 127 deletions(-) create mode 100644 packages/gateway/__tests__/dump/write-run.ts diff --git a/README.md b/README.md index ad61cee5d4..6515b3d7ea 100644 --- a/README.md +++ b/README.md @@ -204,8 +204,10 @@ wraps rendering and writes each observed call once, including calls completed be stage throws. Streaming quantities settle at the deferred exit. Explicit protocol and transport errors take precedence over the settlement's generic failure marker. -Run dumps stream their encoded NDJSON into durable storage with backpressure while appending -the same bytes to a temporary LogStream. Live writes retry at the same byte offset; persistent +Run dumps write through the streaming `DumpStore.putRun` contract with durable storage +backpressure while appending the same encoded NDJSON bytes to a temporary LogStream. +Cold detail reads rehydrate the complete event artifact through `DumpStore.get`. Live writes +retry at the same byte offset; persistent live failure leaves durable recording active. Quiet runs renew their temporary stream and staged-file leases. Closing waits for owned readers and deferred outcomes before publishing completed metadata. Run IDs are assigned at entry; listing remains ordered by completion time diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts index 200ae21f60..3c18b57dbc 100644 --- a/packages/gateway/__tests__/dump/test-fixtures.ts +++ b/packages/gateway/__tests__/dump/test-fixtures.ts @@ -40,7 +40,7 @@ export const eventsOf = (record: StoredDumpRecord): readonly Record JSON.parse(line) as Record); type DumpStubFailMethod = - | 'put' + | 'putRun' | 'list' | 'get' | 'publish' @@ -70,13 +70,9 @@ export const installDumpStubs = ( const store: DumpStore = { async putRun(keyId, run) { const bytes = new Uint8Array(await new Response(run.events).arrayBuffer()); - await store.put(keyId, { meta: await run.metadata, events: bytes }); - }, - async put(keyId, record) { - if (throws.put) throw throws.put; - // A record is already stored-shaped: its stream is encoded once the run is over, so there - // is nothing prepared ahead of the write to rehydrate. - const storedRecord: StoredDumpRecord = record; + const meta = await run.metadata; + if (throws.putRun) throw throws.putRun; + const storedRecord: StoredDumpRecord = { meta, events: bytes }; stored.push({ keyId, record: storedRecord }); const list = records.get(keyId) ?? []; list.unshift(storedRecord); diff --git a/packages/gateway/__tests__/dump/write-run.ts b/packages/gateway/__tests__/dump/write-run.ts new file mode 100644 index 0000000000..3e73326d8c --- /dev/null +++ b/packages/gateway/__tests__/dump/write-run.ts @@ -0,0 +1,9 @@ +import type { DumpStore } from '../../src/dump/store-contract.ts'; +import type { StoredDumpRecord } from '../../src/dump/types.ts'; + +export const writeRun = (store: DumpStore, keyId: string, record: StoredDumpRecord): Promise => store.putRun(keyId, { + id: record.meta.id, + startedAt: record.meta.startedAt, + events: new ReadableStream({ start(controller) { controller.enqueue(record.events); controller.close(); } }), + metadata: Promise.resolve(record.meta), +}); diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts index 3260662bca..2895321ede 100644 --- a/packages/gateway/__tests__/repo/dump-store_test.ts +++ b/packages/gateway/__tests__/repo/dump-store_test.ts @@ -12,6 +12,7 @@ import { initRepo } from '../../src/repo/index.ts'; import { SPILLED_FILE_STAGE_GRACE_MS } from '../../src/repo/spilled-files-policy.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import { collectSpilledFiles } from '../../src/scheduled/spilled-files.ts'; +import { writeRun } from '../dump/write-run.ts'; import { encodeRun, toNdjson, type Facts } from '@floway-dev/pipeline'; import { initFileStore, MemoryFileStore } from '@floway-dev/platform'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; @@ -60,7 +61,7 @@ test('FileDumpStore round-trips a run record as its NDJSON event stream', async const store = new FileDumpStore(db, files); const record = runRecord('01HZZ00000000000000000RUN1', Date.UTC(2026, 5, 1, 12, 0, 0)); - await store.put('key_x', record); + await writeRun(store, 'key_x', record); const fetched = await store.get('key_x', record.meta.id); assertExists(fetched); assertEquals(new TextDecoder().decode(fetched.events), new TextDecoder().decode(record.events)); @@ -81,8 +82,8 @@ test('FileDumpStore lists what a row renders without opening the stream', async const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); const base = Date.UTC(2026, 5, 1, 12, 0, 0); - await store.put('key_x', runRecord('01HZZ00000000000000000RUN1', base)); - await store.put('key_x', runRecord('01HZZ00000000000000000RUN2', base + 1)); + await writeRun(store, 'key_x', runRecord('01HZZ00000000000000000RUN1', base)); + await writeRun(store, 'key_x', runRecord('01HZZ00000000000000000RUN2', base + 1)); const listed = await store.list('key_x', { limit: 10 }); assertEquals(listed.map(meta => meta.id), ['01HZZ00000000000000000RUN2', '01HZZ00000000000000000RUN1']); @@ -99,8 +100,8 @@ test('FileDumpStore retires an expired run record and collects its stream file', initFileStore(files); const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12, 0, 0); - await store.put('key_x', runRecord('01HZZ00000000000000000RUN3', Date.UTC(2026, 5, 1, 9, 0, 0))); - await store.put('key_x', runRecord('01HZZ00000000000000000RUN4', now)); + await writeRun(store, 'key_x', runRecord('01HZZ00000000000000000RUN3', Date.UTC(2026, 5, 1, 9, 0, 0))); + await writeRun(store, 'key_x', runRecord('01HZZ00000000000000000RUN4', now)); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 2 * 3600 }); const originalNow = Date.now; @@ -125,7 +126,7 @@ test('FileDumpStore rejects malformed metadata with its row identity', async () const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); const record = runRecord('01HZZ000000000000000000BADM', Date.UTC(2026, 5, 1, 12)); - await store.put('key_x', record); + await writeRun(store, 'key_x', record); await db.prepare('UPDATE dump_records SET meta_json = ? WHERE key_id = ? AND id = ?') .bind(JSON.stringify({ ...record.meta, upstream: undefined, status: '200' }), 'key_x', record.meta.id) .run(); @@ -138,7 +139,7 @@ test('FileDumpStore rejects a malformed body descriptor before file access', asy const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); const record = runRecord('01HZZ000000000000000000BADD', Date.UTC(2026, 5, 1, 12)); - await store.put('key_x', record); + await writeRun(store, 'key_x', record); await db.prepare('UPDATE dump_records SET response_body_descriptor = ? WHERE key_id = ? AND id = ?') .bind(JSON.stringify({ key: 'dumps/v1/key_x/run.gz', type: 'chunks' }), 'key_x', record.meta.id) .run(); @@ -154,7 +155,7 @@ test('FileDumpStore refuses a row that points at no run stream', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); const record = runRecord('01HZZ000000000000000000NORUN', Date.UTC(2026, 5, 1, 12)); - await store.put('key_x', record); + await writeRun(store, 'key_x', record); await db.prepare('UPDATE dump_records SET response_body_descriptor = ? WHERE key_id = ? AND id = ?') .bind(JSON.stringify({ key: 'dumps/v1/key_x/run.gz', type: 'bytes' }), 'key_x', record.meta.id) .run(); @@ -169,7 +170,7 @@ test('FileDumpStore.list paginates newest-first with the (createdAt, id) cursor' const store = new FileDumpStore(db, files); const base = Date.UTC(2026, 5, 1, 12, 0, 0); for (let i = 0; i < 5; i++) { - await store.put('key_x', runRecord(`01HZZ000000000000000000A0${i}`, base + i)); + await writeRun(store, 'key_x', runRecord(`01HZZ000000000000000000A0${i}`, base + i)); } const first = await store.list('key_x', { limit: 2 }); assertEquals(first.map(m => m.id), ['01HZZ000000000000000000A04', '01HZZ000000000000000000A03']); @@ -186,8 +187,8 @@ test('FileDumpStore applies retention immediately and retires exact expired file const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12, 0, 0); // Old bucket 9:xx, current bucket 12:xx. - await store.put('key_x', runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); - await store.put('key_x', runRecord('01HZZ0000000000000000000A2', now)); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000A2', now)); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 2 * 3600 }); const originalNow = Date.now; Date.now = () => now + 1; @@ -216,7 +217,7 @@ test('growing dump retention can reveal a row not yet physically deleted', async const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12); - await store.put('key_x', runRecord('01HZZ0000000000000000000A4', now - 3 * 3600_000)); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000A4', now - 3 * 3600_000)); const originalNow = Date.now; Date.now = () => now; try { @@ -236,8 +237,8 @@ test('FileDumpStore retires every dump record when retention is disabled and col const files = new MemoryFileStore(); initFileStore(files); const store = new FileDumpStore(db, files); - await store.put('key_x', runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); - await store.put('key_x', runRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0))); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: null }); assertEquals((await store.list('key_x', { limit: 10 })).length, 0); assertEquals(await store.deleteExpiredBatch('key_x', Date.now(), 100), 2); @@ -251,8 +252,8 @@ test('FileDumpStore counts returned dump rows instead of trigger-amplified chang const repo = new SqlRepo(db); const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); - await store.put('key_x', runRecord('01HZZ0000000000000000000C1', Date.UTC(2026, 5, 1, 9))); - await store.put('key_x', runRecord('01HZZ0000000000000000000C2', Date.UTC(2026, 5, 1, 10))); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000C1', Date.UTC(2026, 5, 1, 9))); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000C2', Date.UTC(2026, 5, 1, 10))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: null }); const d1LikeStore = new FileDumpStore(mapRunChangeCount(db, changes => changes * 3), files); @@ -266,8 +267,8 @@ test('FileDumpStore counts returned active dump rows instead of trigger-amplifie const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12); - await store.put('key_x', runRecord('01HZZ0000000000000000000C3', Date.UTC(2026, 5, 1, 9))); - await store.put('key_x', runRecord('01HZZ0000000000000000000C4', Date.UTC(2026, 5, 1, 10))); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000C3', Date.UTC(2026, 5, 1, 9))); + await writeRun(store, 'key_x', runRecord('01HZZ0000000000000000000C4', Date.UTC(2026, 5, 1, 10))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 3600 }); const d1LikeStore = new FileDumpStore(mapRunChangeCount(db, changes => changes * 3), files); @@ -283,11 +284,11 @@ test('a record-ID race leaves only the losing write\'s uniquely keyed files coll initFileStore(files); const store = new FileDumpStore(db, files); const record = runRecord('01HZZ0000000000000000000A3', Date.now()); - await store.put('key_x', record); + await writeRun(store, 'key_x', record); // One file per record now — the run's own stream — so the winner owns one and the loser // staged one under a key of its own. - await expect(store.put('key_x', record)).rejects.toThrow(); + await expect(writeRun(store, 'key_x', record)).rejects.toThrow(); expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(1); expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'staged'").first<{ count: number }>())?.count).toBe(1); @@ -342,7 +343,7 @@ test('FileDumpStore: put + get round-trips through real-filesystem IO', async () const store = new FileDumpStore(db, new TmpDirFileStore(join(root, 'files'))); const record = runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); - await store.put('key_x', record); + await writeRun(store, 'key_x', record); const fetched = await store.get('key_x', '01HZZ0000000000000000000A1'); assertExists(fetched); assertEquals(new TextDecoder().decode(fetched.events), new TextDecoder().decode(record.events)); @@ -359,7 +360,7 @@ test('FileDumpStore filters all retained history before applying the page limit' const record = runRecord(`filter-${i}`, now - i); record.meta.model = i === 4 ? 'needle-model' : 'other'; record.meta.error = i === 4 ? { kind: 'failed', reason: 'socket reset' } : null; - await store.put('key_x', record); + await writeRun(store, 'key_x', record); } expect((await store.list('key_x', { q: 'needle', limit: 1 })).map(meta => meta.id)).toEqual(['filter-4']); expect((await store.list('key_x', { q: 'SOCKET', failures: true, limit: 1 })).map(meta => meta.id)).toEqual(['filter-4']); diff --git a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts index b19cf2fc36..fb40c98a24 100644 --- a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts +++ b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts @@ -1,13 +1,14 @@ import { afterEach, expect, test, vi } from 'vitest'; import { initDumpStore } from '../../src/dump/registry.ts'; -import type { DumpWriteRecord } from '../../src/dump/types.ts'; +import type { StoredDumpRecord } from '../../src/dump/types.ts'; import { FileDumpStore } from '../../src/repo/dump-store.ts'; import { initRepo } from '../../src/repo/index.ts'; import { quantizeOpenAIResponsesRefreshedAt, OPENAI_RESPONSES_REFRESH_GRANULARITY_MS } from '../../src/repo/openai-responses-retention.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import type { ApiKey, StoredOpenAIResponsesItem } from '../../src/repo/types.ts'; import { sweepExpirations } from '../../src/scheduled/expiration-sweeps.ts'; +import { writeRun } from '../dump/write-run.ts'; import { InMemoryRepo } from '../repo/memory.ts'; import { createSqliteTestDb, createSqlJsDatabase, migrationSqlByFilename, wrapSqlJsDatabase } from '../repo/test-sqlite.ts'; import { initFileStore, MemoryFileStore } from '@floway-dev/platform'; @@ -37,7 +38,7 @@ const responseItem = (id: string, refreshedAt: number, apiKeyId = 'key-a'): Stor refreshedAt, }); -const dumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ +const dumpRecord = (id: string, completedAt: number): StoredDumpRecord => ({ meta: { id, startedAt: completedAt - 1, @@ -80,9 +81,9 @@ test('one fair driver drains bounded OpenAI Responses and dump backlogs', async ); await repo.openaiResponsesItems.insertMany([responseItem('msg-current', now)], 0); for (let index = 0; index < 150; index += 1) { - await dumps.put('key-a', dumpRecord(`01K00000000000000000${String(index).padStart(4, '0')}`, dumpExpiredAt)); + await writeRun(dumps, 'key-a', dumpRecord(`01K00000000000000000${String(index).padStart(4, '0')}`, dumpExpiredAt)); } - await dumps.put('key-a', dumpRecord('01K00000000000000000LIVE', now)); + await writeRun(dumps, 'key-a', dumpRecord('01K00000000000000000LIVE', now)); await repo.apiKeys.update('key-a', { dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: OPENAI_RESPONSES_RETENTION_SECONDS }); await sweepExpirations(now); @@ -114,14 +115,14 @@ test('minute ticks catch up with a growing hot dump key without widening a batch await repo.apiKeys.save(key(start)); for (let index = 0; index < 150; index += 1) { - await dumps.put('key-a', dumpRecord(`01K00000000000000001${String(index).padStart(4, '0')}`, start - 3600_001)); + await writeRun(dumps, 'key-a', dumpRecord(`01K00000000000000001${String(index).padStart(4, '0')}`, start - 3600_001)); } for (let tick = 0; tick < 4; tick += 1) { const now = start + tick * 60_000; vi.setSystemTime(now); for (let index = 0; index < 11; index += 1) { - await dumps.put('key-a', dumpRecord(`01K00000000000000002${tick}${String(index).padStart(3, '0')}`, now - 3600_001)); + await writeRun(dumps, 'key-a', dumpRecord(`01K00000000000000002${tick}${String(index).padStart(3, '0')}`, now - 3600_001)); } await sweepExpirations(now); const expected = Math.max(0, 150 + 11 * (tick + 1) - 50 * (tick + 1)); diff --git a/packages/gateway/__tests__/scheduled_test.ts b/packages/gateway/__tests__/scheduled_test.ts index ca1914979d..738adfee13 100644 --- a/packages/gateway/__tests__/scheduled_test.ts +++ b/packages/gateway/__tests__/scheduled_test.ts @@ -1,7 +1,8 @@ import { afterEach, expect, test, vi } from 'vitest'; +import { writeRun } from './dump/write-run.ts'; import { initDumpStore } from '../src/dump/registry.ts'; -import type { DumpWriteRecord } from '../src/dump/types.ts'; +import type { StoredDumpRecord } from '../src/dump/types.ts'; import { FileDumpStore } from '../src/repo/dump-store.ts'; import { initRepo } from '../src/repo/index.ts'; import { SqlRepo } from '../src/repo/sql.ts'; @@ -29,7 +30,7 @@ const apiKey = (id: string, now: number, secretDigit: number): ApiKey => ({ openaiResponsesRetentionSeconds: 0, }); -const fileBackedDumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ +const fileBackedDumpRecord = (id: string, completedAt: number): StoredDumpRecord => ({ meta: { id, startedAt: completedAt - 1, @@ -127,7 +128,7 @@ test('one maintenance tick collects every file retired by its four dump units', const keyId = `key-${keyIndex}`; await repo.apiKeys.save(apiKey(keyId, now, keyIndex + 1)); for (let rowIndex = 0; rowIndex < 50; rowIndex += 1) { - await dumps.put(keyId, fileBackedDumpRecord(`dump-${keyIndex}-${rowIndex}`, now - 3600_001)); + await writeRun(dumps, keyId, fileBackedDumpRecord(`dump-${keyIndex}-${rowIndex}`, now - 3600_001)); } } const { results: ownedFiles } = await db.prepare('SELECT file_key FROM spilled_files ORDER BY file_key') diff --git a/packages/gateway/__tests__/vitest.setup.ts b/packages/gateway/__tests__/vitest.setup.ts index 84a2ff94b7..ac0233291d 100644 --- a/packages/gateway/__tests__/vitest.setup.ts +++ b/packages/gateway/__tests__/vitest.setup.ts @@ -36,7 +36,6 @@ initExecutionCellNamespace(new InProcessExecutionCellNamespace(handleExecutionRe // implementations. const noopStore: DumpStore = { async putRun(_keyId, run) { await new Response(run.events).arrayBuffer(); await run.metadata; }, - async put(): Promise { /* noop */ }, async list(): Promise { return []; }, async get(_keyId: string, _id: DumpRecordId): Promise { return null; }, async deleteExpiredBatch(): Promise { return 0; }, diff --git a/packages/gateway/src/dump/store-contract.ts b/packages/gateway/src/dump/store-contract.ts index f1c8baec47..5c33553008 100644 --- a/packages/gateway/src/dump/store-contract.ts +++ b/packages/gateway/src/dump/store-contract.ts @@ -1,4 +1,4 @@ -import type { DumpMetadata, DumpRecordId, DumpWriteRecord, StoredDumpRecord } from './types.ts'; +import type { DumpMetadata, DumpRecordId, StoredDumpRecord } from './types.ts'; // Per-API-key run storage: metadata in SQL and compressed NDJSON in the // FileStore. Streaming writes stage their file before publishing the row; @@ -19,12 +19,10 @@ export interface DumpRunWrite { } export interface DumpStore { + // Publish metadata only after the complete run artifact exists. Failed + // writes leave staged files collectible by the retention sweep. putRun(keyId: string, run: DumpRunWrite): Promise; - // Write body files BEFORE the metadata row so a partial failure leaves - // orphan files (sweep-collectable), not orphan rows (broken records). - put(keyId: string, record: DumpWriteRecord): Promise; - // Newest-first, paginated by ULID cursor. Reads enforce the API key's // current rolling retention even before queued physical deletion runs. list(keyId: string, opts: DumpListOptions): Promise; diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index 6a1c0f877a..99f0db586c 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -22,7 +22,7 @@ export type DumpUpstreamRef = z.infer; // What went wrong on a failed turn. Either a categorized api-error envelope // (real upstream non-2xx or a gateway-synthesized envelope — `kind` matches // `ApiErrorResult.source`) or an uncategorized failure (anything the -// respond layer caught or observed mid-flight: thrown +// transport edge caught or observed mid-flight: thrown // exceptions, source-emitted error events, downstream cancels, write // errors) carrying its one-line reason text. The categorized form stores // no status — `DumpMetadata.status` already does. @@ -30,32 +30,19 @@ export type DumpErrorMeta = z.infer; export type DumpMetadata = z.infer; -// Canonical protocol frame the gateway records for every -// dump-enabled key. Stored as ProtocolFrame (not the SSE-serialized form) -// so the gateway's live fold and the dashboard's cold fold can share the -// same `collectXProtocolEventsToResult` reducer; the SSE wire view is -// derived on demand by the dashboard via `XProtocolFrameToSSEFrame`. -// -// `unknown` for the event payload because the storage layer is protocol- -// agnostic — the dashboard dispatches the right per-protocol serializer -// based on `meta.path`. +// A protocol frame decoded from a recorded stream. Its producing stage owns +// the event shape; client streams already contain their final wire projection. export type DumpStreamEvent = z.infer; -// The run's NDJSON, still as bytes: it is a body file under the same contract -// as the other two, gzipped into the file store and pointed at by the row's -// descriptor. One `put` carries it whole — measured on production, P99 of a -// turn's request and response together is 2.86 MB, well under the 5 MiB below -// which multipart has nothing to divide. + +// Cold detail reads rehydrate the complete NDJSON artifact. Writes stream the +// same encoded bytes through DumpStore.putRun while metadata settles separately. export type StoredDumpRecord = { meta: DumpMetadata; events: Uint8Array; }; -// A run's stream is encoded once the run is over, so there is nothing to -// compress ahead of the terminal write the way a request body was. -export type DumpWriteRecord = StoredDumpRecord; -// The stored NDJSON verbatim, decoded as UTF-8. One line is one event and one -// SSE `data:` payload, so what a reader parses here is what a live observer -// will parse frame by frame once the fan-out exists. +// The control plane exposes the stored UTF-8 NDJSON verbatim. Live readers +// receive the same bytes through the temporary run stream. export type DumpRecord = { meta: DumpMetadata; events: string; diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index c7f8e0300f..b665d2d3bd 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -12,10 +12,9 @@ import type { DumpMetadata, DumpRecordId, DumpUpstreamRef, - DumpWriteRecord, StoredDumpRecord, } from '../dump/types.ts'; -import { gunzipBytes, gzipBytes, gzipStream } from '../shared/gzip.ts'; +import { gunzipBytes, gzipStream } from '../shared/gzip.ts'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; // Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.{run}.gz`. @@ -64,19 +63,8 @@ const hourBucket = (ms: number): string => { return `${y}${m}${d}${h}`; }; -const bodyPath = (keyId: string, bucket: string, recordId: string, side: 'run'): string => - `${DUMP_FILE_PREFIX}${keyId}/${bucket}/${recordId}-${crypto.randomUUID()}.${side}.gz`; - -const putRawBody = async ( - files: FileStore, - key: string, - rawBytes: Uint8Array, - type: DumpBodyDescriptor['type'], -): Promise => { - const gz = await gzipBytes(rawBytes); - await files.put(key, gz); - return { key, type }; -}; +const bodyPath = (keyId: string, bucket: string, recordId: string): string => + `${DUMP_FILE_PREFIX}${keyId}/${bucket}/${recordId}-${crypto.randomUUID()}.run.gz`; const fetchBody = async (files: FileStore, descriptor: DumpBodyDescriptor): Promise => { const gz = await files.get(descriptor.key); @@ -94,7 +82,7 @@ export class FileDumpStore implements DumpStore { constructor(private readonly db: SqlDatabase, private readonly files: FileStore) {} async putRun(keyId: string, run: DumpRunWrite): Promise { - const fileKey = bodyPath(keyId, hourBucket(run.startedAt), run.id, 'run'); + const fileKey = bodyPath(keyId, hourBucket(run.startedAt), run.id); await this.db.prepare( `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) VALUES (?, 'dump-response', ?, 'staged', ?)`, @@ -132,49 +120,6 @@ export class FileDumpStore implements DumpStore { } finally { stopRenewing(); } } - // Files are staged before writing; rows publish only after the complete run - // artifact exists, so a failed write leaves a collectible orphan file. - async put(keyId: string, record: DumpWriteRecord): Promise { - const bucket = hourBucket(record.meta.completedAt); - const responseFileKey = bodyPath(keyId, bucket, record.meta.id, 'run'); - const staged = [{ fileKey: responseFileKey, ownerKind: 'dump-response' }]; - if (staged.length > 0) { - await this.db - .prepare( - `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) - SELECT - json_extract(value, '$.fileKey'), - json_extract(value, '$.ownerKind'), - json_array(?, ?), - 'staged', - ? - FROM json_each(?)`, - ) - .bind(keyId, record.meta.id, Date.now() + SPILLED_FILE_STAGE_GRACE_MS, JSON.stringify(staged)) - .run(); - } - - const responseDescriptor = await putRawBody(this.files, responseFileKey, record.events, 'run'); - - // Files before row — a partial failure leaves orphan files the sweep - // collects, never an orphan row whose detail fetch would 404. - await this.db.prepare( - `INSERT INTO dump_records - (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, - ).bind( - keyId, - record.meta.id, - record.meta.completedAt, - record.meta.upstream?.id ?? null, - encodePersistedDumpMetadata(record.meta, `dump record ${record.meta.id} metadata`), - NO_EDGE_HEADERS, - null, - null, - encodeDumpBodyDescriptor(responseDescriptor, `dump record ${record.meta.id} response body descriptor`), - ).run(); - } - async list(keyId: string, opts: DumpListOptions): Promise { const beforeId = opts.before ?? null; const beforeRow = beforeId !== null From db0b723749a2ff807845add45fe3e442bfbf29bc Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:40:18 +0800 Subject: [PATCH 137/156] Record the final WebSocket protocol projection Move event-id injection, keepalive sequence shifts, terminal buffering, and original failure envelopes into a declared projection stage. Socket delivery uses prepared JSON packets and selected dump frames match actual wire events. Keep transport keepalives excluded from the protocol record and native count read failures nonbillable. --- .../chat/anthropic-messages/http_test.ts | 21 + .../chat/openai-responses/websocket_test.ts | 37 +- .../data-plane/chat/openai-responses/emit.ts | 13 +- .../chat/openai-responses/pipeline.ts | 14 +- .../openai-responses/project-websocket.ts | 277 ++++++++++++++ .../chat/openai-responses/websocket.ts | 360 ++---------------- packages/gateway/src/dump/run-sink.ts | 7 +- 7 files changed, 363 insertions(+), 366 deletions(-) create mode 100644 packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index e9b60eb08a..0108af05d2 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -250,6 +250,27 @@ test('POST /v1/messages/count_tokens proxies the upstream measurement body', asy assertEquals(await repo.performance.listAll(), []); }); +test('POST /v1/messages/count_tokens preserves a body read failure without charging the native count call', async () => { + const repo = installRepo(); + const fault = new Error('native count body read failed'); + const callAnthropicMessagesCountTokens = vi.fn(async (): Promise => ({ + response: new Response(new ReadableStream({ pull(controller) { controller.error(fault); } }), { headers: { 'content-type': 'application/json' } }), + modelKey: 'count-model', + })); + queueCandidates([makeCandidate({ callAnthropicMessagesCountTokens })]); + const response = await makeApp().request('/v1/messages/count_tokens', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'test-model', max_tokens: 32, messages: [{ role: 'user', content: 'hello' }] }), + }); + assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); + assertEquals(response.status, 500); + const body = await response.text(); + assert(body.includes(fault.message)); + await flushBackground(); + assertEquals(await repo.usage.listAll(), []); + assertEquals(await repo.performance.listAll(), []); +}); + test('POST /v1/messages forwards upstream response headers end-to-end (streaming) and strips hop-by-hop / cookies', async () => { installRepo(); const upstreamHeaders = new Headers({ diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index 144a702b10..e4254931d8 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -12,6 +12,7 @@ import { saveUpstreamForTest } from '../../../repo/upstreams.ts'; import { FakeTime } from '../../../test-time.ts'; import { buildCodexUpstreamRecord, codexModels, copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseOpenAIResponsesResponse, warmModelsForTest } from '../../../test-utils/app.ts'; import { installWorkerWebSocketRuntime, type TestWorkerWebSocket } from '../../../test-utils/worker-websocket.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; import { assert, assertEquals, assertExists, assertStringIncludes, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; const waitForMessages = async ( @@ -301,15 +302,24 @@ test('OpenAI Responses WebSocket dump responseBytes equals the UTF-8 payload byt ); }); -const recordedFrameCount = (record: Parameters[0]): number => { +const recordedClientEvents = (record: Parameters[0]): unknown[] => { const events = eventsOf(runRecordOf(record)); - const selected = events.flatMap(event => { - const facts = event.facts as Record | undefined; - return facts?.['response.chat.clientFrames'] === undefined ? [] : [facts['response.chat.clientFrames'].$stream]; - }).at(-1); + const read = createRunReader(); + const frames = new Map(); + let selected: number | undefined; + for (const event of events) { + const decoded = read(event as unknown as DumpEvent); + const client = decoded?.facts?.['response.chat.clientFrames']; + if (typeof client === 'object' && client !== null && 'stream' in client) selected = client.stream as number; + if (event.type === 'stream.frame') { + const values = frames.get(event.streamId as number) ?? []; + values.push(...(decoded?.frames ?? []).map(frame => (frame as { event: unknown }).event)); + frames.set(event.streamId as number, values); + } + } assertExists(selected); - return events.filter(event => event.type === 'stream.frame' && event.streamId === selected) - .reduce((total, event) => total + (event.frames as readonly unknown[]).length, 0); + assertEquals(events.some(event => event.type === 'stream.end' && event.streamId === selected), true); + return frames.get(selected) ?? []; }; test('OpenAI Responses WebSocket records one frame per event it sent', async () => { @@ -331,7 +341,7 @@ test('OpenAI Responses WebSocket records one frame per event it sent', async () .map(message => JSON.parse(message) as { type?: unknown }) .filter(message => message.type !== KEEP_ALIVE_EVENT_TYPE); assert(sent.length > 0, 'expected the turn to have sent events at all'); - assertEquals(recordedFrameCount(dumps.stored[0]?.record), sent.length); + assertEquals(recordedClientEvents(dumps.stored[0]?.record), sent); } finally { recorded.stop(); client.close(); @@ -412,6 +422,8 @@ test('OpenAI Responses WebSocket rejects the next turn after its API key is rota test('OpenAI Responses WebSocket reports a failed turn when an output item cannot be persisted', async () => { const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); const persistence = vi.spyOn(repo.openaiResponsesItems, 'insertMany').mockRejectedValue(new Error('simulated item persistence failure')); try { await withMockedFetch( @@ -463,6 +475,9 @@ test('OpenAI Responses WebSocket reports a failed turn when an output item canno assertEquals(error.error?.message, 'simulated item persistence failure'); assert(!messages.some(message => message.type === 'response.output_item.done')); assert(!messages.some(isTerminalResponseEvent)); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + assertEquals(recordedClientEvents(dumps.stored[0]?.record), messages); + assertEquals(dumps.stored[0]?.record.meta.error, { kind: 'failed', reason: 'simulated item persistence failure' }); }), ); } finally { @@ -471,7 +486,9 @@ test('OpenAI Responses WebSocket reports a failed turn when an output item canno }); test('OpenAI Responses WebSocket keep-alive waits for the first event and takes a slot in the stream sequence', async () => { - const { apiKey } = await setupAppTest(); + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); // Captured before the clock is faked: the turn's frames cross real event-loop // turns (upstream body reads, item persistence), which a faked `setTimeout` // cannot yield to. @@ -611,6 +628,8 @@ test('OpenAI Responses WebSocket keep-alive waits for the first event and takes ], 'expected the keep-alive to take a slot and shift every later event past it', ); + assert(await drainFramesUntil(() => dumps.stored.length === 1), 'expected the completed turn dump to persist'); + assertEquals(recordedClientEvents(dumps.stored[0]?.record), messages.filter(message => message.type !== KEEP_ALIVE_EVENT_TYPE)); } finally { client.removeEventListener('message', onMessage); } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts index 7f18afa066..3330c5cafe 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts @@ -88,15 +88,8 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra use.gateway, client, ); - // The record is teed above all of that, which is what makes it a record of what the client - // was served rather than of what some layer below had still to rewrite. One tee covers both - // shapes this edge hands out itself, because both read the same iterable: the SSE body, and - // the object the fold assembles from the frames that would have gone out. - // - // Both transports too. A WebSocket turn is the same frames rendered differently, so it is - // recorded here rather than where it is framed — one tee for the family, whatever writes - // what it hands up. Reading is what records, so a transport that stopped early records - // exactly what it took. + // HTTP records the client-facing protocol before writing SSE or folding it into a + // resource. WebSocket records later, after its event-id/sequence/error projection. if (!back['ingress.chat.openaiResponses.wantsStream']) { const frames = recordStream(egress, use.gateway.dump); try { @@ -126,7 +119,7 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra } if (framing === 'events') { const clientStream = completeEventStream(egress, error => { use.gateway.dump?.failed(error); }); - const frames = recordStream(clientStream.frames, use.gateway.dump); + const frames = clientStream.frames; bindClientRelease(back, clientStream.release); return { ...rest, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 901776afaf..c7bbf19b01 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -9,9 +9,6 @@ import { asksForCompaction } from './compaction-policy.ts'; import { emitOpenAIResponses } from './emit.ts'; import { expandShimCompactions } from './expand-compactions.ts'; import { OPENAI_RESPONSES_STREAMED_USAGE, type OpenAIResponsesStreamFraming, type OpenAIResponsesServeEntry, type OpenAIResponsesServeExit } from './facts.ts'; -import { imageGenerationHostedTool } from './hosted-tools/image-generation.ts'; -import { webSearchHostedTool } from './hosted-tools/web-search.ts'; -import { hostedTools } from './hosted-tools.ts'; import { normalizeEmptyToolsForOpenAIResponses } from './normalize-empty-tools-tool-choice.ts'; import { summarizeForCompaction } from './summarize-for-compaction.ts'; import { openaiResponsesNarrowing, openaiResponsesTarget } from './target.ts'; @@ -20,15 +17,19 @@ import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import { composeChat as compose } from '../compose.ts'; import { dialChatWire } from '../dial-wire.ts'; +import { imageGenerationHostedTool } from './hosted-tools/image-generation.ts'; +import { webSearchHostedTool } from './hosted-tools/web-search.ts'; +import { hostedTools } from './hosted-tools.ts'; +import { projectOpenAIResponsesWebSocket, type OpenAIResponsesWebSocketEntry, type OpenAIResponsesWebSocketExit } from './project-websocket.ts'; import type { Pipeline } from '@floway-dev/pipeline'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -export const openaiResponsesServePipeline = ( +export const openaiResponsesServePipeline = ( payload: CanonicalOpenAIResponsesPayload, // SSE is what a run is written in when nothing else claims its frames, which is every // entry over an HTTP body; the WebSocket transport says so because it writes its own. - framing: OpenAIResponsesStreamFraming = 'sse', -): Pipeline => { + framing: Framing = 'sse' as Framing, +): Pipeline => { // One cell per run, written by the stage directly above the one that reads it. The // resolver takes its narrowing at assembly, so this is where the prepared payload crosses // from the membrane to the affinity walk; until the membrane has run, what the client sent @@ -36,6 +37,7 @@ export const openaiResponsesServePipeline = ( let prepared = payload; return compose('openaiResponsesServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), + ...framing === 'events' ? [projectOpenAIResponsesWebSocket] : [], emitOpenAIResponses(payload, framing), hydrateStoredItems(payload, hydrated => { prepared = hydrated; }), resolveChatCandidates(openaiResponsesNarrowing(() => prepared)), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts new file mode 100644 index 0000000000..663fe5fc01 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts @@ -0,0 +1,277 @@ +import type { Fields, OpenAIResponsesServeEntry, OpenAIResponsesServeExit } from './facts.ts'; +import { recordStream, streamReferenceOf } from '../../../dump/run-sink.ts'; +import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../shared/sse.ts'; +import type { ChatServices } from '../services.ts'; +import { bindClientRelease, withClientVerdict } from '../shared/client-stream.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, isOpenAIResponsesTerminalEvent, type ClientOpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { toInternalDebugError } from '@floway-dev/provider'; + +// Our implementor slug prefixes the keep-alive's wire type; the spec reserves +// every unprefixed type for itself, gives `acme:trace_event` as the form, and +// makes `type` and `sequence_number` the only mandatory fields — which is all +// this frame carries. +// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L758-L765 +// A slug type is inert in openai-node's WebSocket reader: the frame is emitted +// under its own literal type name, nothing is listening on that name, and only +// a frame typed `error` is routed into the socket's error path. +// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/resources/responses/ws-base.ts#L346-L371 +export const KEEP_ALIVE_EVENT_TYPE = 'floway:keep_alive'; + +// Extended reasoning turns go completely silent: upstream sends SSE +// `ping` events, `parseOpenAIResponsesStream` drops them, and no frame at +// all reaches this socket for minutes. A silent Workers WebSocket +// does not reliably survive that. Cloudflare states the +// teardown without naming a duration — "when no data is transmitted +// in either direction for a period of time" — and probing a +// `workers.dev` endpoint built on this handler's own `WebSocketPair` +// shape found no constant to design against: from one vantage point +// an idle socket lived a full hour, 4/4, while from another 13 of 16 +// idle sockets died between 215.8 s and 1788.2 s, median around +// 660 s. Teardown is path-dependent and stochastic, and it is always +// a silent EOF — across roughly 40 observed teardowns, not one CLOSE +// frame and not one RST, so the failure carries no protocol-level +// signal. +// https://developers.cloudflare.com/network/websockets/#idle-timeout +// +// Cloudflare's stated remedy, a client-side ping/pong heartbeat, +// does not cover it: on the path that drops, 6 of 9 sockets pinging +// every 30 s died anyway, one of them after 61.5 s. A +// server-originated text frame did cover it — 12/12 survived on that +// same path, with ≤400 s intervals holding and ≥500 s failing. Why a +// data frame outlives a protocol ping there was not established. +// Sending a ping is not open to us regardless: workerd's `WebSocket` +// exposes only accept/send/close/(de)serializeAttachment, and the kj +// layer beneath states the omission as a design decision ("Ping/Pong +// … are not exposed through this interface"). RFC 6455 §5.5.2 is the +// right mechanism, it is unreachable here, and it would not help the +// client that needs it most either, since Codex's frame pump +// swallows control frames and only a text frame rearms its 300 s +// idle timeout. +// https://github.com/cloudflare/workerd/blob/26b5461b7dcc640bb16072f1ba6f2c6df82572ba/src/workerd/api/web-socket.h#L346-L394 +// https://github.com/capnproto/capnproto/blob/e9fa5c7dc98192fc0dc0098ec770db68f997a938/c%2B%2B/src/kj/compat/http.h#L622-L631 +// https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L91-L101 +// https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L695-L699 +// https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/model-provider-info/src/lib.rs#L26 +// +// So the keep-alive is a text frame whose `type` no client +// recognizes. The spec's extension section governs its shape: an +// implementor slug prefix plus a `sequence_number`. A keep-alive is +// neither a delta nor a state-machine event, so it cannot be spelled +// as a `response.*` event; the slug form is what makes it ignorable +// without loss. openai-node's SSE `responses.stream()` helper is the +// one client that treats a prefixed type as fatal — its accumulator +// closes its `switch` on `assertNever` — and it is out of reach here: +// Floway's SSE keep-alive is a comment line, and this frame exists +// only on the WebSocket transport. +// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L758 +// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/lib/responses/ResponseAccumulator.ts#L387-L389 +// +// Held back until the turn's first event has gone out. That gate is +// parity with the stricter transport, not a demand of this one: both +// SDKs' SSE stream helpers refuse anything before `response.created` +// — openai-node rejects even its own `keepalive` type there — while +// every WebSocket reader we can inspect tolerates an unknown type at +// any position, openai-node emitting it under a name nothing listens +// on, openai-python constructing it unchecked, and Codex tracing and +// discarding it. The window before the first event therefore stays +// unprotected, and closing it is a behavior question rather than a +// client-compatibility one. +// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/lib/responses/ResponseAccumulator.ts#L25-L31 +// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/lib/streaming/responses/_responses.py#L369-L370 +// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/resources/responses/responses.py#L4493-L4502 +// https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/sse/responses.rs#L466-L472 + +export interface OpenAIResponsesWebSocketPacket { + readonly text: string; + readonly frame: ProtocolFrame> | null; +} + +export type OpenAIResponsesWebSocketEntry = OpenAIResponsesServeEntry & { + readonly 'ingress.chat.openaiResponses.eventId': string | undefined; +}; +export type OpenAIResponsesWebSocketExit = Omit & { + readonly 'response.chat.openaiResponses.websocket': AsyncIterable; +}; + +export const projectOpenAIResponsesWebSocket = defineStage< + Pick, + Pick, + Fields<'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage' | 'response.http.status' | 'response.chat.clientFrames'> & Pick, + Pick & Fields<'response.chat.openaiResponses.streamedUsage' | 'response.chat.clientFrames'>, + ChatServices +>({ + name: 'projectOpenAIResponsesWebSocket', + through: { + request: { needs: ['ingress.chat.openaiResponses.eventId'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.openaiResponses.rendered', 'response.chat.openaiResponses.streamedUsage', 'response.http.status', 'response.chat.clientFrames'], + consumes: ['response.chat.openaiResponses.rendered', 'response.chat.clientFrames', 'response.chat.openaiResponses.streamedUsage'], + provides: ['response.chat.openaiResponses.websocket', 'response.chat.clientFrames', 'response.chat.openaiResponses.streamedUsage'], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.chat.openaiResponses.rendered': rendered, ...rest } = back; + const eventId = back['ingress.chat.openaiResponses.eventId']; + let started = false; + const verdict = Promise.withResolvers(); + const packet = (event: Record, recorded = true): OpenAIResponsesWebSocketPacket => { + const payload = eventId === undefined ? event : { ...event, event_id: eventId }; + const text = JSON.stringify(payload); + return { text, frame: recorded ? eventFrame(payload) : null }; + }; + const generator = (async function* () { + started = true; + let completed = false; + let failed = false; + try { + if (!(Symbol.asyncIterator in rendered)) { + failed = true; + yield packet({ type: 'error', status: back['response.http.status'], error: normalizeErrorBody(rendered, back['response.http.status']) }); + } else { + const iterator = (rendered as AsyncIterable>)[Symbol.asyncIterator](); + const sequence = createDownstreamSequence(); + let pendingNext = pendingWsFrameResult(iterator.next()); + let terminalEvent: ClientOpenAIResponsesStreamEvent | undefined; + let streamed = false; + let exhausted = false; + try { + while (true) { + if (use.gateway.abortSignal?.aborted) return; + const next = await nextFrameOrKeepAlive(pendingNext); + if (next.type === 'keep-alive') { + if (!streamed) continue; + yield packet({ type: KEEP_ALIVE_EVENT_TYPE, sequence_number: sequence.take() }, false); + continue; + } + if (next.type === 'next-error') throw next.error; + if (next.result.done) { exhausted = true; break; } + const frame = next.result.value; + pendingNext = pendingWsFrameResult(iterator.next()); + if (frame.type !== 'event' || terminalEvent !== undefined) continue; + const event = frame.event; + if (event.type === 'error' || event.type === 'response.failed') failed = true; + // A terminal follows committed item/snapshot writes. Hold it until source + // exhaustion, so a follow-up starts only after all work behind it completed. + if (isOpenAIResponsesTerminalEvent(event)) { terminalEvent = event; continue; } + yield packet(sequence.renumber(event) as unknown as Record); + streamed = true; + } + } finally { + if (!exhausted) await iterator.return?.(); + } + if (terminalEvent === undefined) throw new Error(OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE); + yield packet(sequence.renumber(terminalEvent) as unknown as Record); + } + completed = true; + } catch (error) { + failed = true; + use.gateway.dump?.failed(error); + if (!use.gateway.abortSignal?.aborted) yield packet({ type: 'error', status: 500, error: { ...toInternalDebugError(error), code: 'internal_error' } }); + completed = true; + } finally { + verdict.resolve(failed || !completed); + } + })(); + const packets = recordStream({ [Symbol.asyncIterator]: () => generator }, use.gateway.dump, value => value.frame); + const reference = streamReferenceOf(packets); + bindClientRelease(back, async () => { + if (!started) verdict.resolve(true); + await generator.return(); + }); + return { + ...rest, + 'response.chat.openaiResponses.websocket': move(packets), + 'response.chat.clientFrames': move({ ...reference, async *[Symbol.asyncIterator]() { for await (const value of packets) if (value.frame !== null) yield value.frame; } }), + 'response.chat.openaiResponses.streamedUsage': move(withClientVerdict(back['response.chat.openaiResponses.streamedUsage'], verdict.promise)), + }; + }, +}); + +type WsFrameRaceResult = + | { type: 'frame'; result: IteratorResult> } + | { type: 'next-error'; error: unknown } + | { type: 'keep-alive' }; + +const pendingWsFrameResult = (pendingNext: Promise>>): Promise => + pendingNext.then( + (result): WsFrameRaceResult => ({ type: 'frame', result }), + (error): WsFrameRaceResult => ({ type: 'next-error', error }), + ); + +// The interval is the one already shared with SSE rather than a WebSocket +// constant of its own. Widening the gap between server data frames on a +// dropping path put the boundary between 400 s, which still held the socket +// open, and 500 s, which did not, so 15 s is far more frequent than the +// mechanism needs. It is kept because widening it buys nothing: a keep-alive +// is ~70 bytes, and the earliest unprotected idle teardown seen on that same +// path was 215.8 s, so an interval chosen for economy would spend a real +// margin against a stochastic teardown to save nothing. +const nextFrameOrKeepAlive = async (pendingFrame: Promise): Promise => { + let timeoutId: ReturnType | undefined; + const keepAlive = new Promise(resolve => { + timeoutId = setTimeout(() => resolve({ type: 'keep-alive' }), DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS); + }); + try { + return await Promise.race([pendingFrame, keepAlive]); + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } +}; + +interface DownstreamSequence { + renumber(event: ClientOpenAIResponsesStreamEvent): ClientOpenAIResponsesStreamEvent; + take(): number; +} + +// A WebSocket turn shares one sequence space with the streaming-HTTP events it +// carries, so a keep-alive cannot sit outside that numbering: it takes a real +// slot and every later event is shifted past it. The number is always present +// and always numeric, because a resuming openai-python client compares it with +// `>` against its `starting_after` cursor and `None` there raises a TypeError. +// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L117 +// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/lib/streaming/responses/_responses.py#L59 +// +// Upstream numbering is left untouched until the first keep-alive, and a +// keep-alive can only follow an event that already went out, so the slot it +// takes is always known. +const createDownstreamSequence = (): DownstreamSequence => { + let shift = 0; + let next = 0; + return { + renumber: event => { + if (event.sequence_number === undefined) return event; + const sequenceNumber = event.sequence_number + shift; + next = sequenceNumber + 1; + return { ...event, sequence_number: sequenceNumber }; + }, + take: () => { + shift += 1; + const taken = next; + next += 1; + return taken; + }, + }; +}; + +const normalizeErrorBody = (body: unknown, status: number): Record => { + const source = body && typeof body === 'object' && 'error' in body && typeof (body as { error?: unknown }).error === 'object' + ? (body as { error: Record }).error + : body && typeof body === 'object' + ? body as Record + : {}; + const type = typeof source.type === 'string' + ? source.type + : status >= 500 ? 'server_error' : 'invalid_request_error'; + const message = typeof source.message === 'string' + ? source.message + : `OpenAI Responses request failed with status ${status}.`; + return { + ...source, + type, + code: typeof source.code === 'string' ? source.code : type, + message, + }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index 45bde27cba..2761b01f6b 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -10,9 +10,9 @@ import type { Context } from 'hono'; -import { type OpenAIResponsesFacts, type OpenAIResponsesServeExit } from './facts.ts'; import { createOpenAIResponsesWsSession, type OpenAIResponsesStatefulStore } from './items/store.ts'; import { openaiResponsesServePipeline } from './pipeline.ts'; +import type { OpenAIResponsesWebSocketExit } from './project-websocket.ts'; import { openRunDump } from '../../../dump/run-sink.ts'; import type { RunDump } from '../../../dump/run-sink.ts'; import { apiKeyFromContext, authenticateApiKey, type AuthedContext } from '../../../middleware/auth.ts'; @@ -21,14 +21,12 @@ import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; import { prologueFor, type Ingress } from '../../pipeline/serve.ts'; import type { AttemptState } from '../../shared/gateway-ctx.ts'; import { takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; -import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS, type StreamCompletion } from '../../shared/sse.ts'; +import { type StreamCompletion } from '../../shared/sse.ts'; import type { ChatPrologue } from '../prologue.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; import { SourceStreamState } from '../shared/source-stream-state.ts'; import { move, run } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE } from '@floway-dev/protocols/openai-responses'; import { isOpenAIResponsesTerminalEvent, type CanonicalOpenAIResponsesPayload, type ClientOpenAIResponsesStreamEvent, type OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; import type { ModelCandidate } from '@floway-dev/provider'; import { toInternalDebugError } from '@floway-dev/provider'; @@ -45,17 +43,6 @@ interface OpenAIResponsesWebSocketSocket { const UTF8_ENCODER = new TextEncoder(); -// Our implementor slug prefixes the keep-alive's wire type; the spec reserves -// every unprefixed type for itself, gives `acme:trace_event` as the form, and -// makes `type` and `sequence_number` the only mandatory fields — which is all -// this frame carries. -// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L758-L765 -// A slug type is inert in openai-node's WebSocket reader: the frame is emitted -// under its own literal type name, nothing is listening on that name, and only -// a frame typed `error` is routed into the socket's error path. -// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/resources/responses/ws-base.ts#L346-L371 -export const KEEP_ALIVE_EVENT_TYPE = 'floway:keep_alive'; - interface OpenAIResponsesWebSocketHandlers { onMessage(event: { readonly data: unknown }, socket: OpenAIResponsesWebSocketSocket): void; onClose(event: unknown, socket: OpenAIResponsesWebSocketSocket): void; @@ -365,6 +352,7 @@ const handleClientMessage = async ( 'ingress.chat.sourceProtocol': 'openaiResponses', // A turn on this transport always streams, whatever the client wrote. 'ingress.chat.openaiResponses.wantsStream': true, + 'ingress.chat.openaiResponses.eventId': eventId, 'request.chat.openaiResponses': payload, 'serve.model': payload.model, }) as never, @@ -372,7 +360,7 @@ const handleClientMessage = async ( ); prologue.runDump?.afterRun(drain); - await respondOpenAIResponsesWebSocket({ socket, eventId, signal, isClosed, prologue, facts, drain, turnFailure }); + await respondOpenAIResponsesWebSocket({ socket, signal, isClosed, prologue, facts, drain, turnFailure }); } catch (error) { if (signal.aborted || isClosed()) return; if (error instanceof TranslatorInputError) { @@ -420,334 +408,44 @@ const validateClientMessage = (parsed: unknown): OpenAIResponsesWebSocketClientE const openaiResponsesPayloadFromClientSource = (source: object): CanonicalOpenAIResponsesPayload => ({ ...canonicalizeOpenAIResponsesPayload(source as OpenAIResponsesRequestPayload), stream: true }); -/** The chain hands its answer up at one key, and this entry assembled it to hand up events - * rather than the SSE frames an HTTP body is written from — so what is not a stream at that - * key is the one object arm this transport can see. */ -const isRenderedStream = ( - rendered: OpenAIResponsesFacts['response.chat.openaiResponses.rendered'], -): rendered is AsyncIterable> => Symbol.asyncIterator in rendered; - const respondOpenAIResponsesWebSocket = async (input: { readonly socket: OpenAIResponsesWebSocketSocket; - readonly eventId: string | undefined; readonly signal: AbortSignal; readonly isClosed: () => boolean; readonly prologue: ChatPrologue; - readonly facts: OpenAIResponsesServeExit; + readonly facts: OpenAIResponsesWebSocketExit; readonly drain: () => Promise; readonly turnFailure: OpenAIResponsesWsTurnFailure; }): Promise => { - const { socket, eventId, signal, isClosed, prologue, facts, drain, turnFailure } = input; + const { socket, signal, isClosed, prologue, facts, drain, turnFailure } = input; const ctx = prologue.gateway; - const status = facts['response.http.status']; - const rendered = facts['response.chat.openaiResponses.rendered']; - - if (!isRenderedStream(rendered)) { - // A body at the rendered key is a refusal — the upstream's own, or one the chain made - // before reaching an upstream. Nothing else can arrive there on this transport: a turn - // here always asks to stream, and the one non-refusal body the chain can produce is a - // compaction envelope, which no `response.create` frame has a way to ask for. It goes out - // under the spec's WebSocket error envelope so a client can still compare error.message - // byte-for-byte against upstream. Nothing is left to read, so releasing starts at once. - prologue.services.background(drain()); - turnFailure.fail(status, normalizeErrorBody(rendered, status)); - ctx.dump?.finalize(status, 0); - return; - } - const state = new SourceStreamState(); let completion: StreamCompletion = 'error'; try { - let terminalEvent: ClientOpenAIResponsesStreamEvent | undefined; - const iterator = observeOpenAIResponsesWebSocketFrames(rendered, state)[Symbol.asyncIterator](); - let pendingNext = pendingWsFrameResult(iterator.next()); - let completed = false; - let stoppedByDownstream = false; - let streamed = false; - const sequence = createDownstreamSequence(); - - const stopForDownstream = (): void => { - stoppedByDownstream = true; - completion = 'cancel'; - }; - - try { - while (true) { - if (signal.aborted || isClosed()) { - stopForDownstream(); - return; - } - - const next = await nextFrameOrKeepAlive(pendingNext); - - if (next.type === 'keep-alive') { - // Extended reasoning turns go completely silent: upstream sends SSE - // `ping` events, `parseOpenAIResponsesStream` drops them, and no frame at - // all reaches this socket for minutes. A silent Workers WebSocket - // does not reliably survive that. Cloudflare states the - // teardown without naming a duration — "when no data is transmitted - // in either direction for a period of time" — and probing a - // `workers.dev` endpoint built on this handler's own `WebSocketPair` - // shape found no constant to design against: from one vantage point - // an idle socket lived a full hour, 4/4, while from another 13 of 16 - // idle sockets died between 215.8 s and 1788.2 s, median around - // 660 s. Teardown is path-dependent and stochastic, and it is always - // a silent EOF — across roughly 40 observed teardowns, not one CLOSE - // frame and not one RST, so the failure carries no protocol-level - // signal. - // https://developers.cloudflare.com/network/websockets/#idle-timeout - // - // Cloudflare's stated remedy, a client-side ping/pong heartbeat, - // does not cover it: on the path that drops, 6 of 9 sockets pinging - // every 30 s died anyway, one of them after 61.5 s. A - // server-originated text frame did cover it — 12/12 survived on that - // same path, with ≤400 s intervals holding and ≥500 s failing. Why a - // data frame outlives a protocol ping there was not established. - // Sending a ping is not open to us regardless: workerd's `WebSocket` - // exposes only accept/send/close/(de)serializeAttachment, and the kj - // layer beneath states the omission as a design decision ("Ping/Pong - // … are not exposed through this interface"). RFC 6455 §5.5.2 is the - // right mechanism, it is unreachable here, and it would not help the - // client that needs it most either, since Codex's frame pump - // swallows control frames and only a text frame rearms its 300 s - // idle timeout. - // https://github.com/cloudflare/workerd/blob/26b5461b7dcc640bb16072f1ba6f2c6df82572ba/src/workerd/api/web-socket.h#L346-L394 - // https://github.com/capnproto/capnproto/blob/e9fa5c7dc98192fc0dc0098ec770db68f997a938/c%2B%2B/src/kj/compat/http.h#L622-L631 - // https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L91-L101 - // https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L695-L699 - // https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/model-provider-info/src/lib.rs#L26 - // - // So the keep-alive is a text frame whose `type` no client - // recognizes. The spec's extension section governs its shape: an - // implementor slug prefix plus a `sequence_number`. A keep-alive is - // neither a delta nor a state-machine event, so it cannot be spelled - // as a `response.*` event; the slug form is what makes it ignorable - // without loss. openai-node's SSE `responses.stream()` helper is the - // one client that treats a prefixed type as fatal — its accumulator - // closes its `switch` on `assertNever` — and it is out of reach here: - // Floway's SSE keep-alive is a comment line, and this frame exists - // only on the WebSocket transport. - // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L758 - // https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/lib/responses/ResponseAccumulator.ts#L387-L389 - // - // Held back until the turn's first event has gone out. That gate is - // parity with the stricter transport, not a demand of this one: both - // SDKs' SSE stream helpers refuse anything before `response.created` - // — openai-node rejects even its own `keepalive` type there — while - // every WebSocket reader we can inspect tolerates an unknown type at - // any position, openai-node emitting it under a name nothing listens - // on, openai-python constructing it unchecked, and Codex tracing and - // discarding it. The window before the first event therefore stays - // unprotected, and closing it is a behavior question rather than a - // client-compatibility one. - // https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/lib/responses/ResponseAccumulator.ts#L25-L31 - // https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/lib/streaming/responses/_responses.py#L369-L370 - // https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/resources/responses/responses.py#L4493-L4502 - // https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/sse/responses.rs#L466-L472 - if (!streamed) continue; - if (!sendJson(socket, { type: KEEP_ALIVE_EVENT_TYPE, sequence_number: sequence.take() }, eventId, ctx.dump)) { - stopForDownstream(); - return; - } - continue; - } - if (next.type === 'next-error') throw next.error; - if (next.result.done) { - completed = true; - break; - } - - const frame = next.result.value; - pendingNext = pendingWsFrameResult(iterator.next()); - if (frame.type !== 'event') continue; - - const event = frame.event; - - // The wrapped terminal event arrives only after its item and snapshot - // writes have committed, but the generator still has work to drain - // behind it. Buffer it here and flush it once the loop has run to - // completion, so the terminal event is the last frame of the turn and - // is itself the signal that a follow-up turn may reference this - // response. WebSocket carries the same streaming event objects as - // streaming HTTP, and the specification defines no frame after the - // terminal event: - // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L117 - // Live captures agree. Azure Foundry OpenAI-Responses-over-WebSocket, and the - // ChatGPT backend the Codex CLI talks to, both end a turn on - // `response.completed` and send nothing further while the socket stays - // open and idle. Codex's own reader breaks its loop on that event - // rather than waiting for any trailing envelope: - // https://github.com/openai/codex/blob/acd540f1581bf30f963fccbcce43ac494102242c/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L792-L799 - if (terminalEvent !== undefined) continue; - - if (isOpenAIResponsesTerminalEvent(event)) { - terminalEvent = event; - continue; - } - - if (!sendOpenAIResponsesEvent(socket, sequence.renumber(event), eventId, ctx.dump)) { - stopForDownstream(); - return; - } - streamed = true; - } - } finally { - if (!completed) { - const stopped = iterator.return?.(undefined); - if (stoppedByDownstream) stopped?.catch(() => {}); - else await stopped; - } - } - - if (terminalEvent === undefined) { - throw new Error(OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE); - } - // Renumbered here rather than where it was buffered: keep-alives can still - // fire while the generator drains behind the terminal event, and each of - // those takes a slot that has to land before the terminal event's own. - if (!sendOpenAIResponsesEvent(socket, sequence.renumber(terminalEvent), eventId, ctx.dump)) { - completion = 'cancel'; - return; + for await (const packet of facts['response.chat.openaiResponses.websocket']) { + if (signal.aborted || isClosed() || !sendText(socket, packet.text, ctx.dump)) { completion = 'cancel'; return; } + if (packet.frame?.type !== 'event') continue; + const event = packet.frame.event; + if (event.type === 'error' || event.type === 'response.failed') state.failed = true; + if (isOpenAIResponsesTerminalEvent(event as unknown as ClientOpenAIResponsesStreamEvent) && !state.failed) state.completed = true; } completion = 'eof'; - } catch (error) { - if (signal.aborted || isClosed()) { - completion = 'cancel'; - return; - } - state.failed = true; - turnFailure.fail(500, serverErrorEnvelope(error)); } finally { - // Writing the frames to the socket *is* releasing the body they came from, so the drain - // waits for that to finish. A client that stopped reading still gets here, which is what - // leaves nothing open behind it. await drain(); const failed = state.failedAfter(completion); if (failed) { - // `fail` cannot carry the eviction for every failed turn: one that streamed an `error` - // or `response.failed` terminal answered the client with an event rather than an error - // envelope, and one the client abandoned before the terminal event answered with - // nothing at all. Both settle here. For the abandoned turn the eviction is inert — the - // connection-local cache dies with the socket. turnFailure.evict(); - ctx.dump?.failed(`openai-responses ws turn failed (completion=${completion}, source-failed=${state.failed})`); + ctx.dump?.failed(`openai-responses ws turn failed (completion=${completion}, source-failed=${state.failed})`, { fallback: true }); } - ctx.dump?.finalize(failed ? 500 : 200, 0); - } -}; - -/** Reads the turn's own ending off the frames on their way to the socket. The record is not - * its business: the edge tees these same frames, so what reaches here is already recorded. */ -const observeOpenAIResponsesWebSocketFrames = async function* ( - frames: AsyncIterable>, - state: SourceStreamState, -): AsyncGenerator> { - for await (const frame of frames) { - if (frame.type === 'event') { - const event = frame.event; - const failed = event.type === 'error' || event.type === 'response.failed'; - if (failed) state.failed = true; - if (isOpenAIResponsesTerminalEvent(event) && !failed) state.completed = true; - } - yield frame; - } -}; - -type WsFrameRaceResult = - | { type: 'frame'; result: IteratorResult> } - | { type: 'next-error'; error: unknown } - | { type: 'keep-alive' }; - -const pendingWsFrameResult = (pendingNext: Promise>>): Promise => - pendingNext.then( - (result): WsFrameRaceResult => ({ type: 'frame', result }), - (error): WsFrameRaceResult => ({ type: 'next-error', error }), - ); - -// The interval is the one already shared with SSE rather than a WebSocket -// constant of its own. Widening the gap between server data frames on a -// dropping path put the boundary between 400 s, which still held the socket -// open, and 500 s, which did not, so 15 s is far more frequent than the -// mechanism needs. It is kept because widening it buys nothing: a keep-alive -// is ~70 bytes, and the earliest unprotected idle teardown seen on that same -// path was 215.8 s, so an interval chosen for economy would spend a real -// margin against a stochastic teardown to save nothing. -const nextFrameOrKeepAlive = async (pendingFrame: Promise): Promise => { - let timeoutId: ReturnType | undefined; - const keepAlive = new Promise(resolve => { - timeoutId = setTimeout(() => resolve({ type: 'keep-alive' }), DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS); - }); - try { - return await Promise.race([pendingFrame, keepAlive]); - } finally { - if (timeoutId !== undefined) clearTimeout(timeoutId); + ctx.dump?.finalize(failed ? 500 : facts['response.http.status'], 0); } }; -interface DownstreamSequence { - renumber(event: ClientOpenAIResponsesStreamEvent): ClientOpenAIResponsesStreamEvent; - take(): number; -} - -// A WebSocket turn shares one sequence space with the streaming-HTTP events it -// carries, so a keep-alive cannot sit outside that numbering: it takes a real -// slot and every later event is shifted past it. The number is always present -// and always numeric, because a resuming openai-python client compares it with -// `>` against its `starting_after` cursor and `None` there raises a TypeError. -// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L117 -// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/lib/streaming/responses/_responses.py#L59 -// -// Upstream numbering is left untouched until the first keep-alive, and a -// keep-alive can only follow an event that already went out, so the slot it -// takes is always known. -const createDownstreamSequence = (): DownstreamSequence => { - let shift = 0; - let next = 0; - return { - renumber: event => { - if (event.sequence_number === undefined) return event; - const sequenceNumber = event.sequence_number + shift; - next = sequenceNumber + 1; - return { ...event, sequence_number: sequenceNumber }; - }, - take: () => { - shift += 1; - const taken = next; - next += 1; - return taken; - }, - }; -}; - const serverErrorEnvelope = (error: unknown): Record => ({ ...toInternalDebugError(error), code: 'internal_error', }); -/** The rendered refusal, as this transport's envelope carries one. The chain already answered - * in the words a refusal was made in — the upstream's own body when it sent one, and the - * gateway's own envelope when the refusal was its — so what is left is to state the two - * fields the WebSocket envelope requires of every error whatever wrote it. */ -const normalizeErrorBody = (body: unknown, status: number): Record => { - const source = body && typeof body === 'object' && 'error' in body && typeof (body as { error?: unknown }).error === 'object' - ? (body as { error: Record }).error - : body && typeof body === 'object' - ? body as Record - : {}; - const type = typeof source.type === 'string' - ? source.type - : status >= 500 ? 'server_error' : 'invalid_request_error'; - const message = typeof source.message === 'string' - ? source.message - : `OpenAI Responses request failed with status ${status}.`; - return { - ...source, - type, - code: typeof source.code === 'string' ? source.code : type, - message, - }; -}; - // "WebSocket failures MUST be sent as a JSON `error` envelope with a `status` // code and an `error.code`." // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L166 @@ -764,21 +462,6 @@ const sendError = ( sendJson(socket, { type: 'error', status, error }, eventId, dump); }; -// A turn's own frames go out through this entry, which accepts only a stream -// event whose response resource has already passed the client-facing egress -// stage. The two frames Floway synthesizes for the transport itself — the -// `error` envelope and the `floway:keep_alive` keep-alive — carry no response -// resource at all, and neither belongs to the stream-event union: the error -// envelope is not a streaming event, and the keep-alive is a slug-prefixed -// extension the union deliberately does not model. Both keep the untyped -// `sendJson`. -const sendOpenAIResponsesEvent = ( - socket: OpenAIResponsesWebSocketSocket, - event: ClientOpenAIResponsesStreamEvent, - eventId?: string, - dump?: RunDump | null, -): boolean => sendJson(socket, event, eventId, dump); - const sendJson = ( socket: OpenAIResponsesWebSocketSocket, value: unknown, @@ -789,13 +472,14 @@ const sendJson = ( const payload = eventId === undefined || !value || typeof value !== 'object' ? value : { ...value, event_id: eventId }; - let text: string; - try { - text = JSON.stringify(payload); - socket.send(text); - } catch { - return false; - } + return sendText(socket, JSON.stringify(payload), dump); +}; + +const sendText = (socket: OpenAIResponsesWebSocketSocket, text: string, dump?: RunDump | null): boolean => { + if (socket.readyState !== 1) return false; + try { socket.send(text); } catch { return false; } dump?.recordSentPayloadBytes(UTF8_ENCODER.encode(text).byteLength); return true; }; + +export { KEEP_ALIVE_EVENT_TYPE } from './project-websocket.ts'; diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index c6e833899a..f588df0ad2 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -256,11 +256,11 @@ export interface StreamRecording { * cannot guess and a cast would be it guessing. */ export function recordStream>(stream: AsyncIterable, dump: RunDump | null): AsyncIterable; -export function recordStream(stream: AsyncIterable, dump: RunDump | null, asFrame: (value: T) => ProtocolFrame): AsyncIterable; +export function recordStream(stream: AsyncIterable, dump: RunDump | null, asFrame: (value: T) => ProtocolFrame | null): AsyncIterable; export function recordStream( stream: AsyncIterable, dump: RunDump | null, - asFrame: (value: T) => ProtocolFrame = value => value as ProtocolFrame, + asFrame: (value: T) => ProtocolFrame | null = value => value as ProtocolFrame, ): AsyncIterable { // No recording configured hands the same iterable back, so a record shows no step where // nothing happened and the stream is not wrapped for nobody. @@ -271,7 +271,8 @@ export function recordStream( ...recording.fact, [Symbol.asyncIterator]: () => (async function* () { for await (const value of stream) { - await recording.frame(asFrame(value)); + const frame = asFrame(value); + if (frame !== null) await recording.frame(frame); yield value; } // Reached only where the source ran out on its own, which is what makes the record of From c19eaf2f6b52380ae777673c5061d4f06625f971 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:42:38 +0800 Subject: [PATCH 138/156] fix: collect the recorded client stream with its completion proof Resolve the client stream named by run facts and retain its stream.end state. Normal Chat Completions EOF collects without an invented DONE; a terminal protocol frame alone cannot complete an interrupted recording. Display an incomplete-recording warning independently of protocol failure. Render and copy already-projected Anthropic citation wire fields directly while retaining canonical non-stream reassembly. Remove retired target-capture helpers and refresh run-only metadata documentation. Verified 51 request UI tests, web typecheck, production build/output assertions and scoped lint. --- README.md | 5 +- .../components/requests/detail_test.tsx | 21 ++++- .../components/requests/run-stream_test.ts | 49 ++++++++--- .../components/requests/stream-render_test.ts | 81 ++++++++++++------- apps/web/src/components/requests/detail.tsx | 1 + .../web/src/components/requests/run-stream.ts | 20 +++-- .../src/components/requests/stream-render.ts | 50 ++++-------- apps/web/src/i18n/locales/en.ts | 1 + apps/web/src/i18n/locales/zh-Hans.ts | 1 + .../src/routes/dashboard-monitor-requests.tsx | 8 +- packages/gateway/src/dump/attribution.ts | 39 ++------- packages/gateway/src/repo/dump-store.ts | 14 +--- 12 files changed, 153 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index 6515b3d7ea..74c55df515 100644 --- a/README.md +++ b/README.md @@ -206,8 +206,9 @@ transport errors take precedence over the settlement's generic failure marker. Run dumps write through the streaming `DumpStore.putRun` contract with durable storage backpressure while appending the same encoded NDJSON bytes to a temporary LogStream. -Cold detail reads rehydrate the complete event artifact through `DumpStore.get`. Live writes -retry at the same byte offset; persistent +Cold detail reads rehydrate the complete event artifact through `DumpStore.get`. The Collected +view selects the recorded client stream and uses its `stream.end` event to establish recording +completion. Live writes retry at the same byte offset; persistent live failure leaves durable recording active. Quiet runs renew their temporary stream and staged-file leases. Closing waits for owned readers and deferred outcomes before publishing completed metadata. Run IDs are assigned at entry; listing remains ordered by completion time diff --git a/apps/web/__tests__/components/requests/detail_test.tsx b/apps/web/__tests__/components/requests/detail_test.tsx index 26a512d582..218d085596 100644 --- a/apps/web/__tests__/components/requests/detail_test.tsx +++ b/apps/web/__tests__/components/requests/detail_test.tsx @@ -4,18 +4,24 @@ import { describe, expect, it, vi } from 'vitest'; import { RequestDetailPanel } from '../../../src/components/requests/detail'; import { renderInApp } from '../../render'; import type { DumpRecord } from '@floway-dev/gateway/dump-types'; +import { encodeRun, toNdjson } from '@floway-dev/pipeline'; vi.mock('../../../src/components/ui/body-editor', () => ({ default: ({ text }: { text: string }) =>

{text}
})); const record: DumpRecord = { meta: { id: 'detail', method: 'POST', path: '/v1/chat/completions', startedAt: 0, completedAt: 1, status: 200, upstream: null, model: 'm', inputTokens: null, outputTokens: null, requestBytes: 0, responseBytes: 0, durationMs: 1, error: null }, - events: '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n' - + '{"type":"stage.leaved","stageId":1,"facts":{"response.http.status":200}}\n', + events: toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.http.status': 200 } }, + ])), }; describe('request detail navigation', () => { it('shows run facts, searches events, and opens the collected client response', async () => { renderInApp(); + expect(screen.getByRole('treeitem', { name: 'serve #1' })).toBeTruthy(); + fireEvent.click(screen.getByRole('combobox', { name: 'Response body view' })); + fireEvent.click(screen.getByRole('option', { name: 'Events (2)' })); expect(screen.getByRole('button', { name: '#1 stage.entered serve' })).toBeTruthy(); fireEvent.click(screen.getByRole('button', { name: 'Search events' })); fireEvent.change(screen.getByRole('textbox', { name: 'Search events' }), { target: { value: 'response.http.status' } }); @@ -25,13 +31,22 @@ describe('request detail navigation', () => { expect((await screen.findByTestId('body-content')).textContent).toContain('parsed response'); }); - it('masks credential headers in rendered facts', () => { + it('warns about an incomplete recording while leaving its partial collected response accessible', async () => { + renderInApp(); + expect(screen.getByText('The recorded client stream is incomplete. The collected response may be partial.')).toBeTruthy(); + fireEvent.click(screen.getByRole('combobox', { name: 'Response body view' })); + fireEvent.click(screen.getByRole('option', { name: 'Collected' })); + expect((await screen.findByTestId('body-content')).textContent).toContain('partial response'); + }); + + it('masks credential headers in rendered facts', async () => { const secret = 'Bearer private-request-token-1234567890'; const events = [ { type: 'object', fromObjectId: 1, nodes: [[['authorization', secret]]] }, { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'ingress.http.headers': { $: 1 } } }, ].map(event => `${JSON.stringify(event)}\n`).join(''); const view = renderInApp(); + await screen.findByTestId('body-content'); expect(view.container.textContent).not.toContain(secret); expect(view.container.textContent).toContain('authorization'); expect(view.container.textContent).toContain('••••'); diff --git a/apps/web/__tests__/components/requests/run-stream_test.ts b/apps/web/__tests__/components/requests/run-stream_test.ts index d0a0299b29..0f8a677945 100644 --- a/apps/web/__tests__/components/requests/run-stream_test.ts +++ b/apps/web/__tests__/components/requests/run-stream_test.ts @@ -1,19 +1,42 @@ import { expect, test } from 'vitest'; -import { streamEventsOf } from '../../../src/components/requests/run-stream'; +import { clientStreamOf } from '../../../src/components/requests/run-stream'; import { encodeRun, streamFact, toNdjson } from '@floway-dev/pipeline'; for (const collected of [true, false]) { - test(`selects the client stream among upstream and intermediate frames (${collected ? 'collected' : 'streamed'})`, () => { - const selected = { type: 'stage.leaved' as const, stageId: 1, facts: { 'response.chat.clientFrames': streamFact(3) } }; - const events = toNdjson(encodeRun([ - { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, - { type: 'stream.frame', streamId: 1, frames: [{ type: 'event', event: { text: 'upstream' } }] }, - ...collected ? [] : [selected], - { type: 'stream.frame', streamId: 3, frames: [{ type: 'event', event: { text: 'client' } }] }, - { type: 'stream.frame', streamId: 2, frames: [{ type: 'event', event: { text: 'intermediate' } }] }, - ...collected ? [selected] : [], - ])); - expect(streamEventsOf(events)).toEqual([{ ts: 0, frame: { type: 'event', event: { text: 'client' } } }]); - }); + for (const ended of [true, false]) { + test(`selects client frames and their own completion (${collected ? 'collected' : 'streamed'}, ${ended ? 'ended' : 'interrupted'})`, () => { + const selected = { type: 'stage.leaved' as const, stageId: 1, facts: { 'response.chat.clientFrames': streamFact(3) } }; + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stream.frame', streamId: 1, frames: [{ type: 'event', event: { text: 'upstream' } }] }, + { type: 'stream.end', streamId: 1 }, + ...collected ? [] : [selected], + { type: 'stream.frame', streamId: 3, frames: [{ type: 'event', event: { text: 'client' } }, { type: 'done' }] }, + ...ended ? [{ type: 'stream.end' as const, streamId: 3 }] : [], + { type: 'stream.frame', streamId: 2, frames: [{ type: 'event', event: { text: 'intermediate' } }] }, + { type: 'stream.end', streamId: 2 }, + ...collected ? [selected] : [], + ])); + expect(clientStreamOf(events)).toEqual({ + events: [ + { ts: 0, frame: { type: 'event', event: { text: 'client' } } }, + { ts: 0, frame: { type: 'done' } }, + ], + ended, + }); + }); + } } + +test('keeps a selected empty stream and distinguishes it from no client stream', () => { + expect(clientStreamOf(toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': streamFact(1) } }, + { type: 'stream.end', streamId: 1 }, + ])))).toEqual({ events: [], ended: true }); + expect(clientStreamOf(toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': null } }, + ])))).toBeNull(); +}); diff --git a/apps/web/__tests__/components/requests/stream-render_test.ts b/apps/web/__tests__/components/requests/stream-render_test.ts index 64071b8c97..786f461c36 100644 --- a/apps/web/__tests__/components/requests/stream-render_test.ts +++ b/apps/web/__tests__/components/requests/stream-render_test.ts @@ -1,39 +1,60 @@ -import { describe, expect, it } from 'vitest'; +import { expect, test } from 'vitest'; -import { collectKindFromTargetApi, streamEndedCleanly } from '../../../src/components/requests/stream-render'; -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; +import { clientStreamOf } from '../../../src/components/requests/run-stream'; +import { collectStream, renderStreamEvents, streamEventsCopyText } from '../../../src/components/requests/stream-render'; +import { encodeRun, streamFact, toNdjson, type Event } from '@floway-dev/pipeline'; +import { anthropicMessagesEventToSsePayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -const event = (frame: DumpStreamEvent['frame']): DumpStreamEvent => ({ frame, ts: 1 }); +const recorded = (frames: readonly ProtocolFrame[], ended: boolean) => { + const events: Event[] = [ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': streamFact(1) } }, + { type: 'stream.frame', streamId: 1, frames }, + ...ended ? [{ type: 'stream.end' as const, streamId: 1 }] : [], + ]; + return clientStreamOf(toNdjson(encodeRun(events)))!; +}; -describe('captured stream completion', () => { - it('recognizes the protocol done frame', () => { - expect(streamEndedCleanly([ - event({ type: 'event', event: { value: 'partial' } }), - event({ type: 'done' }), - ])).toBe(true); - }); +const chunk = eventFrame({ id: 'chat-1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { role: 'assistant', content: 'hello' }, finish_reason: 'stop' }] }); - it('marks a recording with no done frame as incomplete', () => { - expect(streamEndedCleanly([ - event({ type: 'event', event: { value: 'partial' } }), - ])).toBe(false); - }); +test('collects normal Chat Completions EOF without inventing a done frame', async () => { + const stream = recorded([chunk], true); + const collected = await collectStream('openai-chat-completions', stream); + expect(collected).toMatchObject({ result: { choices: [{ message: { content: 'hello' } }] }, error: null, truncated: false }); + expect(stream.events).toHaveLength(1); + expect(streamEventsCopyText('openai-chat-completions', stream.events)).not.toContain('[DONE]'); }); -// The upstream (pre-translation) view dispatches its serializer by the TARGET -// protocol the inner attempt spoke, not by the client path (which names the -// source protocol). `collectKindFromTargetApi` mirrors `detectCollectKind` -// for that purpose; null on native turns (no targetApi) and unknown values. -describe('collectKindFromTargetApi', () => { - it('maps each ChatTargetApi to its CollectKind', () => { - expect(collectKindFromTargetApi('anthropicMessages')).toBe('anthropic-messages'); - expect(collectKindFromTargetApi('openaiResponses')).toBe('openai-responses'); - expect(collectKindFromTargetApi('openaiChatCompletions')).toBe('openai-chat-completions'); +for (const terminal of [false, true]) { + test(`marks an interrupted recording incomplete even with ${terminal ? 'a terminal frame' : 'partial output'}`, async () => { + const stream = recorded([chunk, ...terminal ? [{ type: 'done' as const }] : []], false); + expect(await collectStream('openai-chat-completions', stream)).toMatchObject({ error: null, truncated: true }); }); +} - it('returns null for absent or unknown targetApi (native turns)', () => { - expect(collectKindFromTargetApi(null)).toBeNull(); - expect(collectKindFromTargetApi(undefined)).toBeNull(); - expect(collectKindFromTargetApi('unknown')).toBeNull(); - }); +test('separates an ended protocol error from a truncated recording', async () => { + const stream = recorded([eventFrame({ type: 'error', error: { type: 'api_error', message: 'upstream failed' } })], true); + expect(await collectStream('anthropic-messages', stream)).toMatchObject({ result: null, error: expect.stringContaining('upstream failed'), truncated: false }); +}); + +test('renders and copies final Anthropic citation wire fields once while collecting the canonical result', async () => { + const citation = { type: 'search_result_location' as const, url: 'https://example.com/source', title: 'Reference', search_result_index: 0, start_block_index: 0, end_block_index: 1, cited_text: 'evidence' }; + const events: AnthropicMessagesStreamEvent[] = [ + { type: 'message_start', message: { id: 'msg-1', type: 'message', role: 'assistant', content: [], model: 'm', stop_reason: null, stop_sequence: null, usage: { input_tokens: 1, output_tokens: 0 } } }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'answer', citations: [citation] } }, + { type: 'content_block_delta', index: 0, delta: { type: 'citations_delta', citation } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, + { type: 'message_stop' }, + ]; + const wire = events.map(event => eventFrame(anthropicMessagesEventToSsePayload(event))); + const stream = recorded(wire, true); + const rendered = renderStreamEvents('anthropic-messages', stream.events); + expect(JSON.parse(rendered[1]!.text)).toEqual(wire[1]!.event); + expect(JSON.parse(rendered[2]!.text)).toEqual(wire[2]!.event); + const copied = streamEventsCopyText('anthropic-messages', stream.events); + expect(copied).toContain('"source":"https://example.com/source"'); + expect(copied).not.toContain('"url"'); + expect(await collectStream('anthropic-messages', stream)).toMatchObject({ result: { content: [{ type: 'text', text: 'answer', citations: [citation, citation] }] }, error: null, truncated: false }); }); diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index 47a64b3c55..a78d4f8991 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -79,6 +79,7 @@ function RunRecordDetail({ record, collected }: { record: DumpRecord; collected: } label={t('dashboard.requests.exportRecord')} onClick={() => downloadRecords([record])} /> {failure && {failure}} + {collected?.truncated && {t('dashboard.requests.incompleteRecording')}}
{view === 'stages' ? : view === 'events' ? {t('dashboard.requests.events', { count })}} emptyText={t('dashboard.requests.noRunEvents')} /> diff --git a/apps/web/src/components/requests/run-stream.ts b/apps/web/src/components/requests/run-stream.ts index 417b447014..d22b7a1019 100644 --- a/apps/web/src/components/requests/run-stream.ts +++ b/apps/web/src/components/requests/run-stream.ts @@ -1,11 +1,16 @@ import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; +export interface RecordedClientStream { + readonly events: readonly DumpStreamEvent[]; + readonly ended: boolean; +} + // Collected responses consume frames before the edge publishes its identifying fact; // streaming responses consume them afterwards. Resolve the complete record before selecting. -export const streamEventsOf = (ndjson: string): DumpStreamEvent[] => { +export const clientStreamOf = (ndjson: string): RecordedClientStream | null => { const read = createRunReader(); - const streams = new Map(); + const streams = new Map(); let client: number | null = null; for (const line of ndjson.split('\n')) { if (line.length === 0) continue; @@ -15,11 +20,12 @@ export const streamEventsOf = (ndjson: string): DumpStreamEvent[] => { const selected = decoded.facts['response.chat.clientFrames'] as { stream: number } | null; client = selected?.stream ?? null; } - if (event.type === 'stream.frame') { - const bucket = streams.get(event.streamId) ?? []; - for (const frame of decoded?.frames ?? []) bucket.push({ frame: frame as DumpStreamEvent['frame'], ts: 0 }); - streams.set(event.streamId, bucket); + if (event.type === 'stream.frame' || event.type === 'stream.end') { + const stream = streams.get(event.streamId) ?? { events: [], ended: false }; + if (event.type === 'stream.end') stream.ended = true; + else for (const frame of decoded?.frames ?? []) stream.events.push({ frame: frame as DumpStreamEvent['frame'], ts: 0 }); + streams.set(event.streamId, stream); } } - return client === null ? [] : streams.get(client) ?? []; + return client === null ? null : streams.get(client) ?? { events: [], ended: false }; }; diff --git a/apps/web/src/components/requests/stream-render.ts b/apps/web/src/components/requests/stream-render.ts index 3e538fe688..f1114e360d 100644 --- a/apps/web/src/components/requests/stream-render.ts +++ b/apps/web/src/components/requests/stream-render.ts @@ -1,10 +1,8 @@ +import type { RecordedClientStream } from './run-stream'; import { errorMessage } from '../../lib/error-message'; import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; -import { - collectAnthropicMessagesProtocolEventsToResult, - anthropicMessagesProtocolFrameToSSEFrame, -} from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; +import { collectAnthropicMessagesProtocolEventsToResult } from '@floway-dev/protocols/anthropic-messages'; +import { sseFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { collectGeminiGenerateContentProtocolEventsToResult, geminiGenerateContentProtocolFrameToSSEFrame, @@ -48,35 +46,18 @@ export const detectCollectKind = (path: string): CollectKind | null => { return null; }; -// Mirror of `detectCollectKind` for the target protocol of a translated turn. -// The downstream view derives its kind from `meta.path` (the client path = -// source protocol); the upstream view cannot, because the client path names -// the source, not the target. `meta.targetApi` is stamped at capture time by -// `traverseTranslation` and names the target protocol the inner attempt spoke. -export const collectKindFromTargetApi = (api: string | null | undefined): CollectKind | null => { - if (api === 'anthropicMessages') return 'anthropic-messages'; - if (api === 'openaiResponses') return 'openai-responses'; - if (api === 'openaiChatCompletions') return 'openai-chat-completions'; - return null; -}; - -export const streamEndedCleanly = (events: DumpStreamEvent[]): boolean => - events.at(-1)?.frame.type === 'done'; - -const complete = (result: unknown, events: DumpStreamEvent[]): CollectedStream => - ({ result, error: null, truncated: !streamEndedCleanly(events) }); - -export const collectStream = async (kind: CollectKind, events: DumpStreamEvent[]): Promise => { +export const collectStream = async (kind: CollectKind, { events, ended }: RecordedClientStream): Promise => { + const complete = (result: unknown): CollectedStream => ({ result, error: null, truncated: !ended }); try { switch (kind) { case 'openai-chat-completions': - return complete(await collectOpenAIChatCompletionsProtocolEventsToResult(frames(events) as never), events); + return complete(await collectOpenAIChatCompletionsProtocolEventsToResult(frames(events) as never)); case 'anthropic-messages': - return complete(await collectAnthropicMessagesProtocolEventsToResult(frames(events) as never), events); + return complete(await collectAnthropicMessagesProtocolEventsToResult(frames(events) as never)); case 'openai-responses': - return complete(await collectOpenAIResponsesProtocolEventsToResult(frames(events) as never), events); + return complete(await collectOpenAIResponsesProtocolEventsToResult(frames(events) as never)); case 'gemini-generate-content': - return complete(await collectGeminiGenerateContentProtocolEventsToResult(frames(events) as AsyncIterable>), events); + return complete(await collectGeminiGenerateContentProtocolEventsToResult(frames(events) as AsyncIterable>)); case 'openai-completions': { const stream = (async function* () { for (const { frame } of events) { @@ -84,15 +65,15 @@ export const collectStream = async (kind: CollectKind, events: DumpStreamEvent[] if (typed.type === 'event') yield typed.event; } })(); - return complete(await reassembleOpenAICompletionsEvents(stream), events); + return complete(await reassembleOpenAICompletionsEvents(stream)); } } } catch (error) { - return { result: null, error: errorMessage(error), truncated: true }; + return { result: null, error: errorMessage(error), truncated: !ended }; } }; -export const renderStreamEvents = (kind: CollectKind | null, events: DumpStreamEvent[]): RenderedStreamEvent[] => { +export const renderStreamEvents = (kind: CollectKind | null, events: readonly DumpStreamEvent[]): RenderedStreamEvent[] => { return events.map(({ frame, ts }) => { const sse = frameToSse(kind, frame); if (!sse) return { event: null, text: '', parseError: null, timestamp: ts }; @@ -105,14 +86,14 @@ export const renderStreamEvents = (kind: CollectKind | null, events: DumpStreamE }); }; -export const streamEventsCopyText = (kind: CollectKind | null, events: DumpStreamEvent[]): string => { +export const streamEventsCopyText = (kind: CollectKind | null, events: readonly DumpStreamEvent[]): string => { return events.map(({ frame }) => { const sse = frameToSse(kind, frame); return sse ? `${sse.event ? `event: ${sse.event}\n` : ''}data: ${sse.data}\n` : ''; }).filter(Boolean).join('\n'); }; -async function* frames(events: DumpStreamEvent[]) { +async function* frames(events: readonly DumpStreamEvent[]) { for (const event of events) yield event.frame; } @@ -121,7 +102,8 @@ const frameToSse = (kind: CollectKind | null, frame: ProtocolFrame): Ss switch (kind) { case 'openai-chat-completions': return openaiChatCompletionsProtocolFrameToSSEFrame(frame as never, { includeUsageChunk: true }); case 'openai-completions': return openaiCompletionsProtocolFrameToSSEFrame(frame as never); - case 'anthropic-messages': return anthropicMessagesProtocolFrameToSSEFrame(frame as never); + // The client stream already carries Anthropic's citation wire spelling. + case 'anthropic-messages': return frame.type === 'done' ? null : sseFrame(JSON.stringify(frame.event), (frame.event as { type: string }).type); case 'openai-responses': return openaiResponsesProtocolFrameToSSEFrame(frame as never); case 'gemini-generate-content': return geminiGenerateContentProtocolFrameToSSEFrame(frame as never); default: return null; diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index d1f7943b8b..81364f6d80 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -1069,6 +1069,7 @@ const en = { streamView: 'Response body view', noMatches: 'No requests match these filters', collected: 'Collected', + incompleteRecording: 'The recorded client stream is incomplete. The collected response may be partial.', streamEvent: 'Event', eventParseError: 'Invalid JSON', findEvents: 'Search events', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index 6ba9d7378f..14a0427889 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -1021,6 +1021,7 @@ const zhHansCN = { streamView: '响应体视图', noMatches: '没有符合筛选条件的请求', collected: '聚合结果', + incompleteRecording: '记录的客户端流不完整,聚合结果可能只包含部分响应。', streamEvent: '事件', eventParseError: 'JSON 解析失败', findEvents: '搜索事件', diff --git a/apps/web/src/routes/dashboard-monitor-requests.tsx b/apps/web/src/routes/dashboard-monitor-requests.tsx index c3a96c8bb2..c91645add5 100644 --- a/apps/web/src/routes/dashboard-monitor-requests.tsx +++ b/apps/web/src/routes/dashboard-monitor-requests.tsx @@ -10,7 +10,7 @@ import type { ApiKey } from '../api/types'; import { RequestDetailPanel } from '../components/requests/detail'; import { refreshRequestKeys } from '../components/requests/key-refresh'; import { RequestListPanel } from '../components/requests/list'; -import { streamEventsOf } from '../components/requests/run-stream'; +import { clientStreamOf } from '../components/requests/run-stream'; import { collectStream, detectCollectKind, type CollectedStream } from '../components/requests/stream-render'; import { useDumpSubscription } from '../components/requests/use-dump-subscription'; import { DashboardPageHeader } from '../components/ui/dashboard-page-header'; @@ -67,10 +67,8 @@ export async function clientLoader({ request }: Route.ClientLoaderArgs): Promise ]); const record = recordResult?.data ?? null; const collectKind = record ? detectCollectKind(record.meta.path) : null; - // The frames the client was served are in the run's own stream, so the collected view is - // assembled from the record rather than from a second capture beside it. - const streamEvents = record ? streamEventsOf(record.events) : []; - const collected = collectKind && streamEvents.length ? await collectStream(collectKind, streamEvents) : null; + const stream = record ? clientStreamOf(record.events) : null; + const collected = collectKind && stream ? await collectStream(collectKind, stream) : null; return { collected, error: keysResult.error?.message ?? null, diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts index 24ef487c52..ae9f81422f 100644 --- a/packages/gateway/src/dump/attribution.ts +++ b/packages/gateway/src/dump/attribution.ts @@ -1,34 +1,7 @@ -// What a turn's record says about the turn, and what fills it. -// -// `DumpMetadata` is common to both record shapes — the dashboard lists a run -// and a pair of edges in the same list, and a turn's model, upstream and token -// counts do not depend on which mechanism served it — so the hooks that stamp -// attribution and the assembly that turns it into metadata are one thing rather -// than one per shape. -// -// Four independent slots the mid-flight hooks fill: `model` and `upstreamId` -// identify what the turn was about, `inputTokens` / `outputTokens` quantify -// what the upstream reported. They're independent because different outcomes -// set different subsets: -// -// • Every protocol handler calls `requestedModel(model)` immediately after -// parsing the payload, so `model` is set regardless of outcome. -// • `success(identity, usage)` fills all four; the upstream-resolved model -// id may overwrite what `requestedModel` had. -// • `error(kind, upstream?)` records a categorized api-error envelope -// (`kind` matches `ApiErrorResult.source`). Real upstream non-2xx pass -// `upstream` so a 4xx/5xx row in the dashboard names the upstream that -// rejected the call; the gateway arm may also pass it when a candidate -// was already chosen (item-not-found rewrite, hosted-tool input -// rejection). -// • `failed(reason)` records an uncategorized terminal failure: a thrown -// exception, a source-emitted error frame, a downstream cancel, or a -// writer error. Caller passes a string or Error; this one-line-formats it -// (`.message` only — never the stack, which lives in the response body's -// debug envelope). -// -// `requestedModel`-set model survives across both error variants so even an -// outright-failed turn carries model attribution. +// Listing metadata summarizes the run's observed model, upstream and quantities. +// The requested model survives failures before resolution; successful observations +// replace its identity and add only quantities that were actually reported. +// Explicit protocol or stage failures outrank transport and settlement fallbacks. import type { DumpErrorMeta, DumpMetadata, DumpUpstreamRef } from './types.ts'; import { getRepo } from '../repo/index.ts'; @@ -58,7 +31,7 @@ const resolveUpstreamRef = async (id: string | null): Promise Date: Thu, 1 Oct 2026 08:32:03 +0800 Subject: [PATCH 139/156] Carry candidate request projections through immutable facts Read admitted and hydrated requests through declared stage needs. Snapshot pure selected candidate payloads beside selectors, keep provider handles in the registry, and remove request-content cells and getter services. Pipeline constructors now compose static stages; Responses egress borrows its original client request at execution. Verify structural sharing, unsupported candidate exclusion, and dispatch-free projection. --- .../anthropic-messages-count-tokens_test.ts | 12 +++- .../chat/anthropic-messages-pipeline_test.ts | 12 +++- ...mini-generate-content-count-tokens_test.ts | 4 +- .../gemini-generate-content-pipeline_test.ts | 20 +++--- .../openai-chat-completions-pipeline_test.ts | 12 +++- .../chat/openai-responses-compact_test.ts | 4 +- .../chat/openai-responses-pipeline_test.ts | 12 +++- .../chat/openai-responses/emit_test.ts | 6 +- .../hosted-tools/lifecycle_test.ts | 2 +- .../chat/resolve-candidates_test.ts | 64 +++++++++++++++++++ .../data-plane/chat/translated-wires_test.ts | 19 +++--- .../data-plane/pipeline/assembly_test.ts | 14 ++-- .../chat/anthropic-messages/count-tokens.ts | 5 +- .../anthropic-messages/count-tokens/target.ts | 9 +-- .../chat/anthropic-messages/http.ts | 4 +- .../chat/anthropic-messages/pipeline.ts | 5 +- .../chat/anthropic-messages/target.ts | 9 +-- packages/gateway/src/data-plane/chat/facts.ts | 4 ++ .../gemini-generate-content/count-tokens.ts | 5 +- .../count-tokens/target.ts | 9 +-- .../chat/gemini-generate-content/http.ts | 4 +- .../chat/gemini-generate-content/pipeline.ts | 5 +- .../chat/gemini-generate-content/target.ts | 9 +-- .../data-plane/chat/materialize-attempt.ts | 23 ++----- .../chat/openai-chat-completions/http.ts | 2 +- .../chat/openai-chat-completions/pipeline.ts | 5 +- .../chat/openai-chat-completions/target.ts | 9 +-- .../chat/openai-responses/compact.ts | 11 +--- .../data-plane/chat/openai-responses/emit.ts | 9 +-- .../data-plane/chat/openai-responses/http.ts | 4 +- .../openai-responses/hydrate-stored-items.ts | 10 +-- .../chat/openai-responses/pipeline.ts | 13 +--- .../chat/openai-responses/target.ts | 14 +--- .../chat/openai-responses/websocket.ts | 11 +--- .../gateway/src/data-plane/chat/prologue.ts | 20 +----- .../src/data-plane/chat/resolve-candidates.ts | 28 ++++---- .../gateway/src/data-plane/chat/services.ts | 5 -- 37 files changed, 210 insertions(+), 203 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts index 05ae302063..cf7f92a5c9 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts @@ -24,6 +24,14 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal enumerateModelCandidates: vi.fn(), })); +vi.mock('../../../src/data-plane/chat/anthropic-messages/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeAnthropicMessagesAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => affinityPayload }), + }), +})); + let live: readonly ModelCandidate[] = []; type CountTokens = StubChatProviderCall<'anthropicMessagesCountTokens'>; @@ -84,7 +92,7 @@ const count = async ( ) => { const gateway = mockChatGatewayCtx({ wantsStream: false }); return await run( - anthropicMessagesCountTokensPipeline(request), + anthropicMessagesCountTokensPipeline(), move({ 'ingress.http.headers': headers, 'ingress.chat.sourceProtocol': 'anthropicMessages', @@ -96,8 +104,6 @@ const count = async ( background: () => {}, ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), - rememberChatSelection: () => {}, - chatPayloadFor: () => affinityPayload, selectAffinity: () => { throw new Error('a measurement pins nothing; it must not select affinity'); }, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts index b1a86da78b..c3e94163b6 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts @@ -26,6 +26,14 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal enumerateModelCandidates: vi.fn(), })); +vi.mock('../../../src/data-plane/chat/anthropic-messages/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeAnthropicMessagesAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => { asked += 1; return affinityPayload; } }), + }), +})); + let live: readonly ModelCandidate[] = []; type CallAnthropicMessages = StubChatProviderCall<'anthropicMessages'>; @@ -107,7 +115,7 @@ const serveWith = async ( wantsStream: boolean, headers: readonly (readonly [string, string])[] = [], ) => await run( - anthropicMessagesServePipeline(payload), + anthropicMessagesServePipeline(), move({ 'ingress.http.headers': headers, 'ingress.chat.sourceProtocol': 'anthropicMessages', @@ -120,8 +128,6 @@ const serveWith = async ( background: () => {}, ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), - rememberChatSelection: () => {}, - chatPayloadFor: () => { asked += 1; return affinityPayload; }, // Wired where the app wires it: the carrier the edge writes is addressed to whatever the // chain named here. selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts index 737e962b60..1eb4e5bcc2 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts @@ -78,7 +78,7 @@ const count = async ( ) => { const gateway = mockChatGatewayCtx({ wantsStream: false }); return await run( - geminiGenerateContentCountTokensPipeline(request), + geminiGenerateContentCountTokensPipeline(), move({ 'ingress.http.headers': headers, 'ingress.chat.sourceProtocol': 'geminiGenerateContent', @@ -90,8 +90,6 @@ const count = async ( background: () => {}, ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), - rememberChatSelection: () => {}, - chatPayloadFor: () => request, selectAffinity: () => { throw new Error('a measurement pins nothing; it must not select affinity'); }, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts index 5b15b96d18..db48a367e0 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts @@ -28,6 +28,14 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal /** The live candidates the resolver hands back. They never enter the record: a candidate * carries the provider's instance, its fetcher and its models cache, and freezing those is * what putting one in the record would do. What travels is the selector. */ +vi.mock('../../../src/data-plane/chat/gemini-generate-content/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeGeminiGenerateContentAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => { asked += 1; return affinityPayload; } }), + }), +})); + let live: readonly ModelCandidate[] = []; const resolves = (candidates: readonly ModelCandidate[]): void => { @@ -134,15 +142,13 @@ const serve = async (facts: Record) => await serveWith(mockChatGatewayCtx({ wantsStream: facts['ingress.chat.geminiGenerateContent.wantsStream'] === true }), facts); const serveWith = async (gateway: ReturnType, facts: Record) => await run( - geminiGenerateContentServePipeline(payload), + geminiGenerateContentServePipeline(), move(facts) as never, { gateway, background: () => {}, ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), - rememberChatSelection: () => {}, - chatPayloadFor: () => { asked += 1; return affinityPayload; }, // Wired where the app wires it: the carrier the edge writes is addressed to whatever the // chain named here. selectAffinity: (answered: ModelCandidate) => { selected.push(answered); gateway.affinity.select(answered); }, @@ -187,16 +193,12 @@ const affinityCarriers = (events: readonly GeminiGenerateContentResult[]): reado candidate.content.parts.flatMap(part => part.thoughtSignature ?? []))); describe('the Gemini generateContent pipeline', () => { - // The whole entry contract, and it does not mention `request.chat.geminiGenerateContent`: the turn the - // wire translates is not the one the caller handed in but the one materializeAttempt put - // into the record below the fork, which is where a per-candidate payload can exist at all. - // The two `ingress.*` keys are there because the fork declares what the wire under it - // reads — a wire is built against a candidate, so assembly cannot ask one itself. it('assembles, and asks its caller only for what the descending stages need', () => { - expect([...geminiGenerateContentServePipeline(payload).entryNeeds].sort()).toEqual([ + expect([...geminiGenerateContentServePipeline().entryNeeds].sort()).toEqual([ 'ingress.chat.geminiGenerateContent.wantsStream', 'ingress.chat.sourceProtocol', 'ingress.http.headers', + 'request.chat.geminiGenerateContent', 'serve.model', ]); }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts index fef7b7367a..18cb0d2b0a 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts @@ -24,6 +24,14 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal enumerateModelCandidates: vi.fn(), })); +vi.mock('../../../src/data-plane/chat/openai-chat-completions/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeOpenAIChatCompletionsAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => affinityPayload }), + }), +})); + let live: readonly ModelCandidate[] = []; const candidate = ( @@ -101,7 +109,7 @@ const serveWith = async ( wantsStream: boolean, wantsUsageChunk = false, ) => await run( - openaiChatCompletionsServePipeline(payload), + openaiChatCompletionsServePipeline(), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], 'ingress.chat.sourceProtocol': 'openaiChatCompletions', @@ -115,8 +123,6 @@ const serveWith = async ( background: () => {}, ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), - rememberChatSelection: () => {}, - chatPayloadFor: () => affinityPayload, // Wired where the app wires it: the carrier the edge writes is addressed to whatever the // chain named here. selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts index e2ab1ce7b5..a89f72a381 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts @@ -155,7 +155,7 @@ let gateway = mockChatGatewayCtx({ wantsStream: false }); const compact = async (request: CanonicalOpenAIResponsesPayload = payload, signal?: AbortSignal) => { gateway = mockChatGatewayCtx({ wantsStream: false, abortSignal: signal }); const outcome = await run( - openaiResponsesCompactPipeline(request), + openaiResponsesCompactPipeline(), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], 'ingress.chat.sourceProtocol': 'openaiResponses', @@ -167,8 +167,6 @@ const compact = async (request: CanonicalOpenAIResponsesPayload = payload, signa background: () => {}, ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), - rememberChatSelection: () => {}, - chatPayloadFor: () => request, selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts index f89e386ad4..4ea8356ff6 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -33,6 +33,14 @@ vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal enumerateModelCandidates: vi.fn(), })); +vi.mock('../../../src/data-plane/chat/openai-responses/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeOpenAIResponsesAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => { asked += 1; return affinityPayload; } }), + }), +})); + let live: readonly ModelCandidate[] = []; const candidate = ( @@ -161,7 +169,7 @@ const serveWith = async ( wantsStream: boolean, request: CanonicalOpenAIResponsesPayload = payload, ) => await run( - openaiResponsesServePipeline(request), + openaiResponsesServePipeline(), move({ 'ingress.http.headers': [] as readonly (readonly [string, string])[], 'ingress.chat.sourceProtocol': 'openaiResponses', @@ -174,8 +182,6 @@ const serveWith = async ( background: () => {}, ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), - rememberChatSelection: () => {}, - chatPayloadFor: () => { asked += 1; return affinityPayload; }, // Wired where the app wires it: the carrier the edge writes is addressed to whatever the // chain named here. selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts index db60b8ee26..62a44f0bab 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts @@ -22,7 +22,7 @@ const key: ApiKey = { dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0, }; -type Entry = Fields<'ingress.chat.openaiResponses.wantsStream'>; +type Entry = Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>; type Answer = Entry & Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'>; type Exit = Fields<'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage'>; @@ -81,9 +81,9 @@ for (const failureAt of ['before-response', 'upstream-stream', 'snapshot', 'rend }), }); const outcome = await run(compose('clientFailure', [ - emitOpenAIResponses({ model: 'model', input: [], store: false }, 'sse'), + emitOpenAIResponses('sse'), meterUsage('response.chat.openaiResponses.streamedUsage'), dial, - ]), move({ 'ingress.chat.openaiResponses.wantsStream': true }), { gateway, dump: dump.sink }); + ]), move({ 'ingress.chat.openaiResponses.wantsStream': true, 'request.chat.openaiResponses': { model: 'model', input: [], store: false } }), { gateway, dump: dump.sink }); const wire: SseFrame[] = []; for await (const frame of outcome.facts['response.chat.openaiResponses.rendered'] as AsyncIterable) wire.push(frame); expect((await outcome.facts['response.chat.openaiResponses.streamedUsage'])?.failed).toBe(true); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts index bbc444f948..cbbcdd4ae6 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts @@ -58,7 +58,7 @@ for (const client of ['unread', 'partial-before-tool', 'partial-second', 'comple }; const [selector] = base.services.rememberCandidates([candidate]); const executed = await run(compose, Record>('hostedLifecycle', [ - emitOpenAIResponses(payload, 'sse'), hostedTools([registration], { streamedUsage: key, targetOf: () => 'openaiResponses' }), ...openaiResponsesWire(key), + emitOpenAIResponses('sse'), hostedTools([registration], { streamedUsage: key, targetOf: () => 'openaiResponses' }), ...openaiResponsesWire(key), ]), move({ 'request.chat.openaiResponses': payload, 'route.attempt': selector, 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'openaiResponses', 'ingress.chat.openaiResponses.wantsStream': true, 'serve.usage.prior': [] }), { ...base.services, gateway, recordProtocolFrames: (frames: AsyncIterable) => frames, selectAffinity: () => {}, }); diff --git a/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts b/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts new file mode 100644 index 0000000000..f2cbd77158 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts @@ -0,0 +1,64 @@ +import { expect, test, vi } from 'vitest'; + +import type { ChatFacts } from '../../../src/data-plane/chat/facts.ts'; +import { resolveChatCandidates } from '../../../src/data-plane/chat/resolve-candidates.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +test('resolution publishes pure candidate request projections without provider handles or dispatching an untried candidate', async () => { + const first = stubModelCandidate(); + const second = stubModelCandidate({ model: { id: 'second' } }); + const unsupported = stubModelCandidate({ model: { id: 'unsupported', endpoints: {} } }); + const payload = { model: 'model', messages: [{ role: 'user' as const, content: 'question' }] }; + const changed = { ...payload, messages: [{ role: 'user' as const, content: 'projected' }] }; + const projectFirst = vi.fn(() => payload); + const projectSecond = vi.fn(() => changed); + const evaluated: unknown[] = []; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [first, second, unsupported], sawModel: true, failedUpstreams: [] }); + const resolver = resolveChatCandidates({ + requestKey: 'request.chat.openaiChatCompletions', + canServe: candidate => candidate !== unsupported, + affinity: async actual => { + expect(actual).toBe(payload); + return { + requiredTargets: [], + evaluateCandidate: candidate => { + evaluated.push(candidate); + return { kind: 'accepted', degrades: false, preferred: true, materialize: candidate === first ? projectFirst : projectSecond }; + }, + }; + }, + unsupported: model => model, + refuse: () => ({}), + refuses: [], + }); + const dispatch = vi.fn(); + const terminal = defineStage, Record>({ + name: 'observeResolution', + return: { provides: ['response.usage.billable', 'response.http.headers', 'response.http.body'] }, + execute: async facts => move({ ...facts, 'response.usage.billable': [], 'response.http.headers': [], 'response.http.body': null }), + }); + const registry = createCandidateRegistry(); + const outcome = await run(compose<{ 'serve.model': string } & Pick, Record>('resolve', [resolver, terminal]), move({ + 'serve.model': 'model', 'request.chat.openaiChatCompletions': payload, + }), { gateway: mockChatGatewayCtx(), ...registry, httpCall: dispatch }); + const values = outcome.facts['request.chat.candidatePayloads'] as Readonly>; + expect(values[0]).toBe(payload); + expect(values[1]).toBe(changed); + expect(Object.keys(values)).toEqual(['0', '1']); + expect(Object.isFrozen(values)).toBe(true); + expect(Object.isFrozen(first.provider.instance)).toBe(false); + expect(evaluated).toEqual([first, second]); + expect(projectFirst).toHaveBeenCalledTimes(1); + expect(projectSecond).toHaveBeenCalledTimes(1); + expect(dispatch).not.toHaveBeenCalled(); + await outcome.drain(); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index 95fc615a25..f6581218fb 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -169,7 +169,6 @@ const contextExceeded = async (): Promise( pipeline: Pipeline, facts: Record, - attemptPayload: unknown, ) => { const gateway = mockChatGatewayCtx({ wantsStream: false }); return await run(pipeline, move(facts) as never, { @@ -177,8 +176,6 @@ const serve = async ( background: () => {}, ...chatFixtureHttpServices(gateway), ...createCandidateRegistry(), - rememberChatSelection: () => {}, - chatPayloadFor: () => attemptPayload, selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); @@ -194,41 +191,41 @@ const openaiResponsesPayload = { model: MODEL, input: [{ type: 'message', role: const geminiGenerateContentPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiGenerateContentPayload; const serveOpenAIChatCompletions = async (payload: OpenAIChatCompletionsPayload = openaiChatCompletionsPayload) => - await serve(openaiChatCompletionsServePipeline(payload), { + await serve(openaiChatCompletionsServePipeline(), { 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'openaiChatCompletions', 'ingress.chat.openaiChatCompletions.wantsStream': false, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false, 'request.chat.openaiChatCompletions': payload, 'serve.model': MODEL, - }, payload); + }); const serveAnthropicMessages = async (payload: AnthropicMessagesPayload = anthropicMessagesPayload) => - await serve(anthropicMessagesServePipeline(payload), { + await serve(anthropicMessagesServePipeline(), { 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'anthropicMessages', 'ingress.chat.anthropicMessages.wantsStream': false, 'request.chat.anthropicMessages': payload, 'serve.model': MODEL, - }, payload); + }); const serveOpenAIResponses = async (payload: CanonicalOpenAIResponsesPayload = openaiResponsesPayload) => - await serve(openaiResponsesServePipeline(payload), { + await serve(openaiResponsesServePipeline(), { 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'openaiResponses', 'ingress.chat.openaiResponses.wantsStream': false, 'request.chat.openaiResponses': payload, 'serve.model': MODEL, - }, payload); + }); const serveGeminiGenerateContent = async (payload: GeminiGenerateContentPayload = geminiGenerateContentPayload) => - await serve(geminiGenerateContentServePipeline(payload), { + await serve(geminiGenerateContentServePipeline(), { 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'geminiGenerateContent', 'ingress.chat.geminiGenerateContent.wantsStream': false, 'request.chat.geminiGenerateContent': payload, 'serve.model': MODEL, - }, payload); + }); // ── What the client was answered with, per protocol ─────────────────────────────────────── diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index d20ad8ad02..6aa0d9ab7f 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -28,13 +28,13 @@ const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] ['OpenAI Completions', () => openaiCompletionsServePipeline], ['OpenAI Audio Transcriptions', () => openaiAudioTranscriptionServePipeline], ['alpha search', () => searchServePipeline({ kind: 'search' } as never)], - ['OpenAI Chat Completions', () => openaiChatCompletionsServePipeline({ model: 'm', messages: [] } as never)], - ['Anthropic Messages', () => anthropicMessagesServePipeline({ model: 'm', messages: [] } as never)], - ['Gemini generateContent', () => geminiGenerateContentServePipeline({ model: 'm', contents: [] } as never)], - ['OpenAI Responses', () => openaiResponsesServePipeline({ model: 'm', input: [] } as never)], - ['Anthropic Messages count_tokens', () => anthropicMessagesCountTokensPipeline({ model: 'm', messages: [] } as never)], - ['Gemini generateContent countTokens', () => geminiGenerateContentCountTokensPipeline({ model: 'm', contents: [] } as never)], - ['OpenAI Responses compact', () => openaiResponsesCompactPipeline({ model: 'm', input: [] } as never)], + ['OpenAI Chat Completions', () => openaiChatCompletionsServePipeline()], + ['Anthropic Messages', () => anthropicMessagesServePipeline()], + ['Gemini generateContent', () => geminiGenerateContentServePipeline()], + ['OpenAI Responses', () => openaiResponsesServePipeline()], + ['Anthropic Messages count_tokens', () => anthropicMessagesCountTokensPipeline()], + ['Gemini generateContent countTokens', () => geminiGenerateContentCountTokensPipeline()], + ['OpenAI Responses compact', () => openaiResponsesCompactPipeline()], ]; describe('every family assembles', () => { diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index f43cf28266..7e0f44c9d3 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -8,9 +8,8 @@ import { narrowing } from './count-tokens/target.ts'; import { anthropicMessagesCountTokensWire } from './count-tokens/wire.ts'; import { failover } from '../../pipeline/failover.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -export const anthropicMessagesCountTokensPipeline = (payload: AnthropicMessagesPayload): Pipeline => +export const anthropicMessagesCountTokensPipeline = (): Pipeline => compose('anthropicMessagesCountTokens', [ // A measurement goes through settlement like every other run. It provides an empty // billed set because nothing here is billable today, not because the operation is @@ -18,7 +17,7 @@ export const anthropicMessagesCountTokensPipeline = (payload: AnthropicMessagesP // nothing else would change. writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, undefined, false), emitAnthropicMessagesTokenCount, - resolveChatCandidates(narrowing(payload)), + resolveChatCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), owns: ['response.http.body'], diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/target.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/target.ts index 7d699ac5ca..0dd6517ec8 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/target.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/target.ts @@ -2,19 +2,16 @@ import type { Counted } from './facts.ts'; import type { ChatNarrowing } from '../../resolve-candidates.ts'; import { chatTargetPicker } from '../../shared/target-picker.ts'; import { analyzeAnthropicMessagesAffinity } from '../affinity/ingress.ts'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; /** Counting has no translation path: only an upstream's own Anthropic Messages endpoint can answer the * question, so a candidate that would serve generation over a translated wire cannot serve * this. */ export const anthropicMessagesCountTokensTarget = chatTargetPicker(['anthropicMessages']); -/** A candidate that cannot serve *this* request is not a candidate — and what the client's - * own turn carries decides the order the rest are tried in, which is why the narrowing is - * built from the request rather than being a constant. */ -export const narrowing = (payload: AnthropicMessagesPayload): ChatNarrowing> => ({ +export const narrowing: ChatNarrowing, 'request.chat.anthropicMessages'> = ({ + requestKey: 'request.chat.anthropicMessages', canServe: candidate => anthropicMessagesCountTokensTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeAnthropicMessagesAffinity(payload, gateway.affinity.codec), + affinity: async (payload, gateway) => await analyzeAnthropicMessagesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /messages/count_tokens endpoint.`, refuse: (status, message) => ({ 'response.chat.anthropicMessages': { status, message } }), refuses: ['response.chat.anthropicMessages'], diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts index 0b9ebcec65..8fa4433b28 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts @@ -80,7 +80,7 @@ export const anthropicMessagesHttp = { return await serveThrough( c, prologue, - anthropicMessagesServePipeline(payload), + anthropicMessagesServePipeline(), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'anthropicMessages', @@ -118,7 +118,7 @@ export const anthropicMessagesHttp = { return await serveThrough( c, prologue, - anthropicMessagesCountTokensPipeline(payload), + anthropicMessagesCountTokensPipeline(), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'anthropicMessages', diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index e2dce273da..8893ba9799 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -13,13 +13,12 @@ import { runAnthropicMessagesWebSearchTool } from './web-search-tool.ts'; import { dialChatWire } from '../dial-wire.ts'; import { anthropicMessagesWireFor } from './wires.ts'; import type { Pipeline } from '@floway-dev/pipeline'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -export const anthropicMessagesServePipeline = (payload: AnthropicMessagesPayload): Pipeline => +export const anthropicMessagesServePipeline = (): Pipeline => compose('anthropicMessagesServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), emitAnthropicMessages, - resolveChatCandidates(narrowing(payload)), + resolveChatCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), owns: ['response.http.body'], diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/target.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/target.ts index b0b55e6ee8..a1f06e4ce7 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/target.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/target.ts @@ -2,18 +2,15 @@ import type { ChatNarrowing } from '../resolve-candidates.ts'; import type { Fields } from './facts.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; /** `/v1/messages` prefers its own wire, then the translated OpenAI Responses path, then the * translated OpenAI Chat Completions path. */ export const anthropicMessagesTarget = chatTargetPicker(['anthropicMessages', 'openaiResponses', 'openaiChatCompletions']); -/** A candidate that cannot serve *this* request is not a candidate — and what the client's - * own turn carries decides the order the rest are tried in, which is why the narrowing is - * built from the request rather than being a constant. */ -export const narrowing = (payload: AnthropicMessagesPayload): ChatNarrowing> => ({ +export const narrowing: ChatNarrowing, 'request.chat.anthropicMessages'> = ({ + requestKey: 'request.chat.anthropicMessages', canServe: candidate => anthropicMessagesTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeAnthropicMessagesAffinity(payload, gateway.affinity.codec), + affinity: async (payload, gateway) => await analyzeAnthropicMessagesAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /messages endpoint.`, refuse: (status, message) => ({ 'response.chat.anthropicMessages': { status, message }, diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts index de8b3fc202..8eb90606ad 100644 --- a/packages/gateway/src/data-plane/chat/facts.ts +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -21,6 +21,8 @@ import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-respon * and no target role. */ export type ChatSourceProtocol = 'openaiChatCompletions' | 'anthropicMessages' | 'openaiResponses' | 'geminiGenerateContent'; +export type ChatRequestKey = 'request.chat.openaiChatCompletions' | 'request.chat.anthropicMessages' | 'request.chat.openaiResponses' | 'request.chat.geminiGenerateContent'; + export interface ChatFacts extends GatewayFacts { /** Which protocol the client spoke. Read by the stages that must know — a vendor * normalizer genuinely does — rather than reached for through an ambient field. @@ -39,6 +41,8 @@ export interface ChatFacts extends GatewayFacts { * always asks the upstream for one; this decides who is shown it. */ 'ingress.chat.openaiChatCompletions.wantsUsageChunk': boolean; + 'request.chat.candidatePayloads': Readonly>; + /** One key per protocol. A translation consumes one and provides another, which is the * whole of what a protocol handoff is. */ 'request.chat.openaiChatCompletions': OpenAIChatCompletionsPayload; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index 0b5e9caeff..174d3aa41c 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -12,9 +12,8 @@ import { stripUnsupportedToolsFromGeminiGenerateContent } from './strip-unsuppor import { failover } from '../../pipeline/failover.ts'; import { anthropicMessagesCountTokensWire } from '../anthropic-messages/count-tokens/wire.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; -import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -export const geminiGenerateContentCountTokensPipeline = (payload: GeminiGenerateContentPayload): Pipeline => +export const geminiGenerateContentCountTokensPipeline = (): Pipeline => compose('geminiGenerateContentCountTokens', [ // A measurement goes through settlement like every other run. It provides an empty // billed set because nothing here is billable today, not because the operation is @@ -22,7 +21,7 @@ export const geminiGenerateContentCountTokensPipeline = (payload: GeminiGenerate // nothing else would change. writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, undefined, false), emitGeminiGenerateContentTokenCount, - resolveChatCandidates(narrowing(payload)), + resolveChatCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), owns: ['response.http.body'], diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/target.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/target.ts index dfb1762117..e1bec55b7c 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/target.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/target.ts @@ -2,18 +2,15 @@ import type { Counted } from './facts.ts'; import type { ChatNarrowing } from '../../resolve-candidates.ts'; import { chatTargetPicker } from '../../shared/target-picker.ts'; import { analyzeGeminiGenerateContentAffinity } from '../affinity/ingress.ts'; -import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; /** Counting is reachable only over an upstream's own Anthropic Messages endpoint: no other protocol * answers the question, and no translation invents an answer. */ export const geminiGenerateContentCountTokensTarget = chatTargetPicker(['anthropicMessages']); -/** A candidate that cannot serve *this* request is not a candidate — and what the client's - * own turn carries decides the order the rest are tried in, which is why the narrowing is - * built from the request rather than being a constant. */ -export const narrowing = (payload: GeminiGenerateContentPayload): ChatNarrowing> => ({ +export const narrowing: ChatNarrowing, 'request.chat.geminiGenerateContent'> = ({ + requestKey: 'request.chat.geminiGenerateContent', canServe: candidate => geminiGenerateContentCountTokensTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeGeminiGenerateContentAffinity(payload, gateway.affinity.codec), + affinity: async (payload, gateway) => await analyzeGeminiGenerateContentAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support countTokens.`, refuse: (status, message) => ({ 'response.chat.geminiGenerateContent': { status, message } }), refuses: ['response.chat.geminiGenerateContent'], diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts index c6fac4ea25..125bb956d0 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts @@ -95,7 +95,7 @@ const runGeminiGenerateContentGenerate = async (c: AuthedContext, model: string, return await serveThrough( c, prologue, - geminiGenerateContentServePipeline(payload), + geminiGenerateContentServePipeline(), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'geminiGenerateContent', @@ -128,7 +128,7 @@ const runGeminiGenerateContentCountTokens = async (c: AuthedContext, model: stri return await serveThrough( c, prologue, - geminiGenerateContentCountTokensPipeline(payload), + geminiGenerateContentCountTokensPipeline(), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'geminiGenerateContent', diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index e80708838a..5689194b08 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -15,13 +15,12 @@ import { stripUnsupportedToolsFromGeminiGenerateContent } from './strip-unsuppor import { dialChatWire } from '../dial-wire.ts'; import { geminiGenerateContentWireFor } from './wires.ts'; import type { Pipeline } from '@floway-dev/pipeline'; -import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -export const geminiGenerateContentServePipeline = (payload: GeminiGenerateContentPayload): Pipeline => +export const geminiGenerateContentServePipeline = (): Pipeline => compose('geminiGenerateContentServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), emitGeminiGenerateContent, - resolveChatCandidates(narrowing(payload)), + resolveChatCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), owns: ['response.http.body'], diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/target.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/target.ts index b8b5893a8a..065e738ad9 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/target.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/target.ts @@ -2,18 +2,15 @@ import type { ChatNarrowing } from '../resolve-candidates.ts'; import type { Fields } from './facts.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; -import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; /** `:generateContent` has no wire of its own, so the whole preference list is translated: * OpenAI Chat Completions first, then Anthropic Messages, then OpenAI Responses. */ export const geminiGenerateContentTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); -/** A candidate that cannot serve *this* request is not a candidate — and what the client's - * own turn carries decides the order the rest are tried in, which is why the narrowing is - * built from the request rather than being a constant. */ -export const narrowing = (payload: GeminiGenerateContentPayload): ChatNarrowing> => ({ +export const narrowing: ChatNarrowing, 'request.chat.geminiGenerateContent'> = ({ + requestKey: 'request.chat.geminiGenerateContent', canServe: candidate => geminiGenerateContentTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeGeminiGenerateContentAffinity(payload, gateway.affinity.codec), + affinity: async (payload, gateway) => await analyzeGeminiGenerateContentAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the Gemini generateContent endpoint.`, refuse: (status, message) => ({ 'response.chat.geminiGenerateContent': { status, message }, diff --git a/packages/gateway/src/data-plane/chat/materialize-attempt.ts b/packages/gateway/src/data-plane/chat/materialize-attempt.ts index c303d450c7..f23e59adee 100644 --- a/packages/gateway/src/data-plane/chat/materialize-attempt.ts +++ b/packages/gateway/src/data-plane/chat/materialize-attempt.ts @@ -1,21 +1,12 @@ +import type { Chat } from './facts.ts'; import type { ChatServices } from './services.ts'; import type { Slice } from './stage-contracts.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -/** - * Puts the payload this attempt is owed into the record. - * - * Affinity materializes one per candidate — client-carried state rewritten for the upstream - * that will see it — and it has to enter the record here rather than be read by the stage - * that dials, because everything between this stage and the dial rewrites the request as a - * fact. An ending that asked the resolver instead would be reading a payload no rule - * had touched, and the rewrites would go nowhere. - * - * It sits below the fork because there is no single payload above it: each candidate has its - * own, and re-running the suffix is what produces the next one. - */ +// Select this attempt's affinity projection below the fork. Rules then rewrite that request +// fact, while the resolver's other candidate values remain shared for a later attempt. export const materializeAttempt = (requestKey: string) => defineStage< - Slice<'route.attempt'>, + Slice<'route.attempt'> & Chat<'request.chat.candidatePayloads'>, Record, Record, Record, @@ -23,9 +14,9 @@ export const materializeAttempt = (requestKey: string) => defineStage< >({ name: `materializeAttempt:${requestKey}`, through: { - request: { needs: ['route.attempt'], consumes: [], provides: [requestKey] }, + request: { needs: ['route.attempt', 'request.chat.candidatePayloads'], consumes: [], provides: [requestKey] }, response: { needs: [], consumes: [], provides: [] }, }, - execute: async (facts, next, use) => - await next({ ...facts, [requestKey]: move(use.chatPayloadFor(facts['route.attempt'])) }), + execute: async (facts, next) => + await next({ ...facts, [requestKey]: move(facts['request.chat.candidatePayloads'][facts['route.attempt'].candidateId]) }), }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts index 16bf9eec73..1d8b4221dc 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts @@ -54,7 +54,7 @@ export const openaiChatCompletionsHttp = { return await serveThrough( c, prologue, - openaiChatCompletionsServePipeline(payload), + openaiChatCompletionsServePipeline(), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'openaiChatCompletions', diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index 90e1258ea7..f09fe1e664 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -11,13 +11,12 @@ import { normalizeEmptyToolsForOpenAIChatCompletions } from './normalize-empty-t import { dialChatWire } from '../dial-wire.ts'; import { openaiChatCompletionsWireFor } from './wires.ts'; import type { Pipeline } from '@floway-dev/pipeline'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -export const openaiChatCompletionsServePipeline = (payload: OpenAIChatCompletionsPayload): Pipeline => +export const openaiChatCompletionsServePipeline = (): Pipeline => compose('openaiChatCompletionsServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), emitOpenAIChatCompletions, - resolveChatCandidates(narrowing(payload)), + resolveChatCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.openaiChatCompletions'?: unknown })['response.chat.openaiChatCompletions']), owns: ['response.http.body'], diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/target.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/target.ts index 5f7b1e4098..6048b68b96 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/target.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/target.ts @@ -2,18 +2,15 @@ import type { ChatNarrowing } from '../resolve-candidates.ts'; import type { Fields } from './facts.ts'; import { chatTargetPicker } from '../shared/target-picker.ts'; import { analyzeOpenAIChatCompletionsAffinity } from './affinity/ingress.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; /** `/v1/chat/completions` prefers its own wire, then the translated Anthropic Messages path, * then the translated OpenAI Responses path. */ export const openaiChatCompletionsTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); -/** A candidate that cannot serve *this* request is not a candidate — and what the client's - * own turn carries decides the order the rest are tried in, which is why the narrowing is - * built from the request rather than being a constant. */ -export const narrowing = (payload: OpenAIChatCompletionsPayload): ChatNarrowing> => ({ +export const narrowing: ChatNarrowing, 'request.chat.openaiChatCompletions'> = ({ + requestKey: 'request.chat.openaiChatCompletions', canServe: candidate => openaiChatCompletionsTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeOpenAIChatCompletionsAffinity(payload, gateway.affinity.codec), + affinity: async (payload, gateway) => await analyzeOpenAIChatCompletionsAffinity(payload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /chat/completions endpoint.`, refuse: (status, message, reason) => ({ 'response.chat.openaiChatCompletions.streamedUsage': null, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index f22ef1cdb6..17463c36de 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -18,18 +18,13 @@ import { openaiResponsesNarrowing, openaiResponsesTarget } from './target.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import type { Pipeline } from '@floway-dev/pipeline'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -export const openaiResponsesCompactPipeline = (payload: CanonicalOpenAIResponsesPayload): Pipeline => { - // One cell per run, written by the stage directly above the one that reads it — the same - // hand-across the generate chain makes, for the same reason: the resolver takes its - // narrowing at assembly, before any fact exists. - let prepared = payload; +export const openaiResponsesCompactPipeline = (): Pipeline => { return compose('openaiResponsesCompact', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), emitOpenAIResponsesCompaction, - hydrateStoredItems(payload, hydrated => { prepared = hydrated; }), - resolveChatCandidates(openaiResponsesNarrowing(() => prepared)), + hydrateStoredItems, + resolveChatCandidates(openaiResponsesNarrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), owns: ['response.http.body'], diff --git a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts index 3330c5cafe..9dc020c8bd 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts @@ -29,16 +29,16 @@ import { toInternalDebugError } from '@floway-dev/provider'; * Only the last step differs: `renderSSE` is a wire format written over an HTTP body, and a * transport that frames each event itself takes the events it would have been written from. */ -export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, framing: OpenAIResponsesStreamFraming) => defineStage< - Fields<'ingress.chat.openaiResponses.wantsStream'>, - Fields<'ingress.chat.openaiResponses.wantsStream'>, +export const emitOpenAIResponses = (framing: OpenAIResponsesStreamFraming) => defineStage< + Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>, + Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>, Fields<'ingress.chat.openaiResponses.wantsStream' | 'response.chat.openaiResponses' | 'response.http.headers' | 'response.chat.openaiResponses.streamedUsage'>, Fields<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.openaiResponses.streamedUsage'>, ChatServices >({ name: 'emitOpenAIResponses', through: { - request: { needs: ['ingress.chat.openaiResponses.wantsStream'], consumes: [], provides: [] }, + request: { needs: ['ingress.chat.openaiResponses.wantsStream', 'request.chat.openaiResponses'], consumes: [], provides: [] }, response: { needs: ['response.chat.openaiResponses', 'response.http.headers', OPENAI_RESPONSES_STREAMED_USAGE], consumes: ['response.chat.openaiResponses', 'response.http.headers'], @@ -46,6 +46,7 @@ export const emitOpenAIResponses = (client: CanonicalOpenAIResponsesPayload, fra }, }, execute: async (facts, next, use) => { + const client = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; const back = await next(facts); const { 'response.chat.openaiResponses': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what diff --git a/packages/gateway/src/data-plane/chat/openai-responses/http.ts b/packages/gateway/src/data-plane/chat/openai-responses/http.ts index 8b175f628b..085d634cc0 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/http.ts @@ -84,7 +84,7 @@ export const openaiResponsesHttp = { return await serveThrough( c, prologue, - openaiResponsesServePipeline(payload), + openaiResponsesServePipeline(), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'openaiResponses', @@ -115,7 +115,7 @@ export const openaiResponsesHttp = { return await serveThrough( c, prologue, - openaiResponsesCompactPipeline(payload), + openaiResponsesCompactPipeline(), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'openaiResponses', diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts b/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts index 0511e88074..e153a79348 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts @@ -42,10 +42,7 @@ const refuseStoredItems = ( * nowhere to be routed to. Both of its refusals are answers it already holds, which is why * it carries the `return` trait alongside `through`. */ -export const hydrateStoredItems = ( - client: CanonicalOpenAIResponsesPayload, - prepared: (payload: CanonicalOpenAIResponsesPayload) => void, -) => defineStage< +export const hydrateStoredItems = defineStage< Fields<'request.chat.openaiResponses'>, Fields<'request.chat.openaiResponses' | 'request.chat.openaiResponses.privatePayloads'>, Record, @@ -92,7 +89,7 @@ export const hydrateStoredItems = ( // Both lists: what the turn now names is read by id, and what the client itself sent is // read by item hash as well, so a body repeated verbatim finds the row it already made. - await store.loadInputItems(expanded.input, client.input); + await store.loadInputItems(expanded.input, asked.input); let hydrated: ReturnType; try { @@ -108,9 +105,8 @@ export const hydrateStoredItems = ( // The client's own input, not the expansion's `item_reference` prefix: the prefix is // already in the snapshot this turn inherited, and staging it again would repeat it. - await store.stageInputItems(client.input); + await store.stageInputItems(asked.input); - prepared(hydrated.payload); return await next({ ...facts, 'request.chat.openaiResponses': move(hydrated.payload), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index c7bbf19b01..60765c5969 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -22,25 +22,18 @@ import { webSearchHostedTool } from './hosted-tools/web-search.ts'; import { hostedTools } from './hosted-tools.ts'; import { projectOpenAIResponsesWebSocket, type OpenAIResponsesWebSocketEntry, type OpenAIResponsesWebSocketExit } from './project-websocket.ts'; import type { Pipeline } from '@floway-dev/pipeline'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; export const openaiResponsesServePipeline = ( - payload: CanonicalOpenAIResponsesPayload, // SSE is what a run is written in when nothing else claims its frames, which is every // entry over an HTTP body; the WebSocket transport says so because it writes its own. framing: Framing = 'sse' as Framing, ): Pipeline => { - // One cell per run, written by the stage directly above the one that reads it. The - // resolver takes its narrowing at assembly, so this is where the prepared payload crosses - // from the membrane to the affinity walk; until the membrane has run, what the client sent - // is the whole of what is known about the turn. - let prepared = payload; return compose('openaiResponsesServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), ...framing === 'events' ? [projectOpenAIResponsesWebSocket] : [], - emitOpenAIResponses(payload, framing), - hydrateStoredItems(payload, hydrated => { prepared = hydrated; }), - resolveChatCandidates(openaiResponsesNarrowing(() => prepared)), + emitOpenAIResponses(framing), + hydrateStoredItems, + resolveChatCandidates(openaiResponsesNarrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), owns: ['response.http.body'], diff --git a/packages/gateway/src/data-plane/chat/openai-responses/target.ts b/packages/gateway/src/data-plane/chat/openai-responses/target.ts index 7fc225a561..4dd2a3a8a1 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/target.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/target.ts @@ -8,18 +8,10 @@ import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/open * the translated OpenAI Chat Completions path. */ export const openaiResponsesTarget = chatTargetPicker(['openaiResponses', 'anthropicMessages', 'openaiChatCompletions']); -/** A candidate that cannot serve *this* request is not a candidate — and what the client's - * own turn carries decides the order the rest are tried in, which is why the narrowing is - * built from the request rather than being a constant. - * - * It reads the request through a function because the one it has to read is the *prepared* - * one: a `previous_response_id` continuation carries the prior turn's state on items the - * client never sent, and a turn is pinned by what its items carry. The narrowing is built - * at assembly, before any fact exists, so the membrane hands the prepared payload across - * through the run's own cell rather than through the record. */ -export const openaiResponsesNarrowing = (prepared: () => CanonicalOpenAIResponsesPayload): ChatNarrowing> => ({ +export const openaiResponsesNarrowing: ChatNarrowing, 'request.chat.openaiResponses'> = ({ + requestKey: 'request.chat.openaiResponses', canServe: candidate => openaiResponsesTarget.canServe(candidate.model.endpoints), - affinity: async gateway => await analyzeOpenAIResponsesAffinity(prepared(), gateway.affinity.codec), + affinity: async (payload, gateway) => await analyzeOpenAIResponsesAffinity(payload as CanonicalOpenAIResponsesPayload, gateway.affinity.codec), unsupported: model => `Model ${model} does not support the /responses endpoint.`, refuse: (status, message, reason) => ({ 'response.chat.openaiResponses.streamedUsage': null, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index 2761b01f6b..c9183fd077 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -28,7 +28,6 @@ import { SourceStreamState } from '../shared/source-stream-state.ts'; import { move, run } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; import { isOpenAIResponsesTerminalEvent, type CanonicalOpenAIResponsesPayload, type ClientOpenAIResponsesStreamEvent, type OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; -import type { ModelCandidate } from '@floway-dev/provider'; import { toInternalDebugError } from '@floway-dev/provider'; import { canonicalizeOpenAIResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; @@ -236,20 +235,12 @@ const openOpenAIResponsesWebSocketTurn = ( }, turn.store); const base = prologueFor(gateway, { body: turn.body, headers: turn.headers }, dump); - let materialize: ((candidate: ModelCandidate) => unknown) | undefined; return { ...base, gateway, services: { ...base.services, gateway, - rememberChatSelection: payloadFor => { materialize = payloadFor; }, - chatPayloadFor: selector => { - if (materialize === undefined) { - throw new Error('chatPayloadFor: nothing was resolved in this run; the selector did not come from it'); - } - return materialize(base.services.resolveAttempt(selector)); - }, selectAffinity: candidate => { gateway.affinity.select(candidate); }, }, }; @@ -346,7 +337,7 @@ const handleClientMessage = async ( const { facts, drain } = await run( // The transport frames its own answer, so the edge hands up the events rather than the // SSE a body would have been written from. - openaiResponsesServePipeline(payload, 'events'), + openaiResponsesServePipeline('events'), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'openaiResponses', diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts index 8c2b578c8f..276ce462b0 100644 --- a/packages/gateway/src/data-plane/chat/prologue.ts +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -1,21 +1,11 @@ -// Opening a chat run. The seam every non-chat family uses builds the services a run is -// given; a chat run needs three more, and this is where they come from. -// -// All three are the live half of resolution. A `ModelCandidate` never enters the record — -// `move()` would freeze the provider's own models cache — and neither does the payload -// affinity materializes for one, because materializing rewrites client-carried state for the -// upstream that will see it and the result is per candidate. So the resolver keeps them and -// the stage that dials asks for them back by selector, which is the same shape the shared -// services already use for the candidate itself. +// Chat admission adds the request-owned store and affinity ports to shared run services. import type { ChatServices } from './services.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from './shared/gateway-ctx.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import type { ApiKey } from '../../repo/types.ts'; -import type { AttemptSelector } from '../pipeline/facts.ts'; import { gatewayCtxOptions, prologueFor, runDumpOf, type Ingress, type Prologue } from '../pipeline/serve.ts'; import type { OpenAIResponsesStatefulStore } from './openai-responses/items/store.ts'; -import type { ModelCandidate } from '@floway-dev/provider'; export interface ChatPrologue extends Prologue { readonly services: ChatServices; @@ -41,20 +31,12 @@ export const openChatPrologue = ( const gateway = createChatGatewayCtxFromHono(c, ctxOptions, options.storeFactory); const base = prologueFor(gateway, ingress, runDumpOf(ctxOptions)); - let materialize: ((candidate: ModelCandidate) => unknown) | undefined; return { ...base, gateway, services: { ...base.services, gateway, - rememberChatSelection: payloadFor => { materialize = payloadFor; }, - chatPayloadFor: (selector: AttemptSelector) => { - if (materialize === undefined) { - throw new Error('chatPayloadFor: nothing was resolved in this run; the selector did not come from it'); - } - return materialize(base.services.resolveAttempt(selector)); - }, selectAffinity: candidate => { gateway.affinity.select(candidate); }, }, }; diff --git a/packages/gateway/src/data-plane/chat/resolve-candidates.ts b/packages/gateway/src/data-plane/chat/resolve-candidates.ts index 79cfffc366..dcb4e4ccce 100644 --- a/packages/gateway/src/data-plane/chat/resolve-candidates.ts +++ b/packages/gateway/src/data-plane/chat/resolve-candidates.ts @@ -1,3 +1,4 @@ +import type { ChatFacts, ChatRequestKey } from './facts.ts'; import type { ChatServices } from './services.ts'; import { enumerateModelCandidates } from '../providers/resolution.ts'; import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; @@ -13,14 +14,15 @@ import type { ModelCandidate } from '@floway-dev/provider'; export type ChatRefusal = 'routing-unavailable' | 'model-unsupported' | 'model-missing'; /** How a source protocol narrows and orders the candidates that could serve it. */ -export interface ChatNarrowing { +export interface ChatNarrowing { + readonly requestKey: RequestKey; /** Which upstream wires this source prefers, in order. A candidate none of whose endpoints * appear here cannot serve the request whatever else it offers. */ readonly canServe: (candidate: ModelCandidate) => boolean; /** What the client's own turn says about where it must go. Client-carried state — a * an OpenAI Responses `previous_response_id`, an encrypted reasoning blob — pins the turn to the * upstream that issued it, so this runs before any candidate is tried. */ - readonly affinity: (gateway: ChatGatewayCtx) => Promise>; + readonly affinity: (payload: Pick[RequestKey], gateway: ChatGatewayCtx) => Promise>; readonly unsupported: (model: string) => string; readonly refuse: (status: number, message: string, reason: ChatRefusal) => Refusal; readonly refuses: readonly (keyof Refusal)[]; @@ -34,9 +36,9 @@ export interface ChatNarrowing { * nowhere to go — and that is an answer this stage already holds, which is why it carries * the `return` trait alongside `through`. */ -export const resolveChatCandidates = (narrowing: ChatNarrowing) => defineStage< - Slice<'serve.model'>, - Slice<'serve.model' | 'serve.candidates'>, +export const resolveChatCandidates = (narrowing: ChatNarrowing) => defineStage< + Slice<'serve.model'> & Pick, + Slice<'serve.model' | 'serve.candidates'> & Pick & Pick, Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'> & Refusal, @@ -44,7 +46,7 @@ export const resolveChatCandidates = (narrowing: ChatNar >({ name: 'resolveChatCandidates', through: { - request: { needs: ['serve.model'], consumes: [], provides: ['serve.candidates'] }, + request: { needs: ['serve.model', narrowing.requestKey], consumes: [], provides: ['serve.candidates', 'request.chat.candidatePayloads'] }, response: { needs: ['response.usage.billable', 'response.http.headers'], consumes: [], provides: [] }, }, return: { provides: ['response.usage.billable', 'response.http.headers', 'response.http.body', ...narrowing.refuses] }, @@ -69,7 +71,7 @@ export const resolveChatCandidates = (narrowing: ChatNar ...narrowing.refuse(status, message, reason), }); - const affinity = await narrowing.affinity(use.gateway); + const affinity = await narrowing.affinity(facts[narrowing.requestKey] as ChatFacts[RequestKey], use.gateway); const viable = candidates.filter(candidate => narrowing.canServe(candidate)); const selection = selectAffinityCandidates(viable, affinity); // A turn whose carried state needs two upstreams at once is a request the client can fix @@ -83,11 +85,13 @@ export const resolveChatCandidates = (narrowing: ChatNar return refuse(sawModel ? 400 : 404, appendFailedUpstreams(missing, failedUpstreams), sawModel ? 'model-unsupported' : 'model-missing'); } - // The live half stays with the resolver; only selectors travel. The payload a candidate - // is owed is part of that live half — affinity materializes it per candidate, and it - // carries the client's own state rewritten for the upstream that will see it. - use.rememberChatSelection(selection.payloadFor); + const selectors = use.rememberCandidates(selection.candidates); + // Selection and payload projection are pure after carrier decoding. Snapshot only the + // accepted candidates' request values; live provider instances remain in the registry. + const payloads = Object.fromEntries(selectors.map((selector, index) => [ + selector.candidateId, selection.payloadFor(selection.candidates[index]), + ])); await use.log.debug('resolved chat candidates', { model, viable: selection.candidates.length, resolved: candidates.length }); - return await next({ ...facts, 'serve.candidates': move(use.rememberCandidates(selection.candidates)) }); + return await next({ ...facts, 'serve.candidates': move(selectors), 'request.chat.candidatePayloads': move(payloads) }); }, }); diff --git a/packages/gateway/src/data-plane/chat/services.ts b/packages/gateway/src/data-plane/chat/services.ts index c3e46d0379..7523f99cb1 100644 --- a/packages/gateway/src/data-plane/chat/services.ts +++ b/packages/gateway/src/data-plane/chat/services.ts @@ -1,13 +1,8 @@ import type { GatewayServices } from '../pipeline/services.ts'; import type { ChatGatewayCtx } from './shared/gateway-ctx.ts'; -import type { AttemptSelector } from '../pipeline/facts.ts'; import type { ModelCandidate, ProviderChatServices } from '@floway-dev/provider'; -/** What a chat family needs beyond the shared services: the affinity selection this run - * made, so the stage that dials can ask for the payload the winning candidate is owed. */ export interface ChatServices extends GatewayServices, ProviderChatServices { readonly gateway: ChatGatewayCtx; - readonly rememberChatSelection: (payloadFor: (candidate: ModelCandidate) => unknown) => void; - readonly chatPayloadFor: (selector: AttemptSelector) => unknown; readonly selectAffinity: (candidate: ModelCandidate) => void; } From 8a8134f8dd415461599d0647e8984c30eb6def08 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 08:43:23 +0800 Subject: [PATCH 140/156] fix: transfer body release before finite response readers adopt it Text and byte readers take responsibility for exhausting or failing the owned HTTP body before reading starts. A read rejection now propagates its original error without a second locked-stream cancellation failure; actual endpoint call facts remain available to settlement. Verified 14 focused tests, owning provider typechecks and scoped lint. Native primitive read rejections separately exposed missing failure context and were handed to the core owner. --- .../rewrite-context-window-error_test.ts | 28 +++++++++++++++++++ .../rewrite-context-window-error.ts | 5 ++-- .../__tests__/stages/pipelines_test.ts | 25 ++++++++++++++++- .../provider/src/stages/decode-response.ts | 6 ++-- 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 packages/provider-copilot/__tests__/stages/anthropic-messages/rewrite-context-window-error_test.ts diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/rewrite-context-window-error_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/rewrite-context-window-error_test.ts new file mode 100644 index 0000000000..20f7926adb --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/rewrite-context-window-error_test.ts @@ -0,0 +1,28 @@ +import { expect, test } from 'vitest'; + +import { rewriteCopilotContextWindowError } from '../../../src/stages/anthropic-messages/rewrite-context-window-error.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose, getFailureFacts, move, run } from '@floway-dev/pipeline'; +import { decodeProviderResponse, observeProviderCall, type ProviderChatServices } from '@floway-dev/provider'; +import { noopUpstreamCallOptions } from '@floway-dev/test-utils'; + +test('failed finite error-body reading preserves its original failure and the actual call facts', async () => { + const error = new Error('error response body failed'); + const body = new ReadableStream({ start(controller) { controller.error(error); } }); + const services: ProviderChatServices = { + httpCall: () => ({ ...noopUpstreamCallOptions(), signal: undefined, fetcher: async () => new Response(body, { status: 400 }) }), + recordProtocolFrames: frames => frames, + }; + const entry = move({ + 'request.provider.modelKey': 'claude-test', + 'request.http.callId': 1, + 'request.http.url': 'https://copilot.example/v1/messages', + 'request.http.method': 'POST', + 'request.http.headers': [], + 'request.http.body': {}, + 'request.http.encoding': 'json' as const, + }); + const pipeline = compose('copilotErrorBody', [rewriteCopilotContextWindowError, decodeProviderResponse('anthropicMessages'), observeProviderCall, http]); + await expect(run(pipeline, entry, services)).rejects.toBe(error); + expect(getFailureFacts(error)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'claude-test' }); +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts b/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts index 2c32fb034c..f846e1d43f 100644 --- a/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts @@ -13,9 +13,10 @@ export const rewriteCopilotContextWindowError = defineStage= 200 && exchange.status < 300) return move({ ...back, 'response.copilot.errorBody': null }); - const bytes = new Uint8Array(await exchangeResponse(exchange).arrayBuffer()); - const text = new TextDecoder().decode(bytes); + const response = exchangeResponse(exchange); if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const bytes = new Uint8Array(await response.arrayBuffer()); + const text = new TextDecoder().decode(bytes); let body: unknown; try { body = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; body = text; } const tooLong = text.includes('Request body is too large for model context window') || text.includes('context_length_exceeded'); diff --git a/packages/provider-custom/__tests__/stages/pipelines_test.ts b/packages/provider-custom/__tests__/stages/pipelines_test.ts index 8b9949acb6..4b2d0ea6c5 100644 --- a/packages/provider-custom/__tests__/stages/pipelines_test.ts +++ b/packages/provider-custom/__tests__/stages/pipelines_test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import { createCustomPipelines } from '../../src/pipelines.ts'; -import { move, run, type Event } from '@floway-dev/pipeline'; +import { getFailureFacts, move, run, type Event } from '@floway-dev/pipeline'; import { providerModelFacts, type ProviderChatServices, type ProviderOperationRequest } from '@floway-dev/provider'; import { jsonResponse, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; @@ -56,3 +56,26 @@ test('count_tokens forwards typed beta intent and retains raw HTTP response fact await executed.drain(); }); }); + +for (const operation of ['openaiChatCompletions', 'anthropicMessagesCountTokens'] as const) { + test(`finite response body failure preserves its original error and call facts (${operation})`, async () => { + const error = new Error('response body socket failed'); + const body = new ReadableStream({ start(controller) { controller.error(error); } }); + const pipeline = pipelines[operation]; + if (pipeline === undefined) throw new Error(`Missing ${operation} chain`); + const input = move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: 'wire-model' })), + 'request.provider.payload': { max_tokens: 12, messages: [] }, + 'request.http.callId': 1, + 'request.http.headers': [], + 'request.provider.anthropicBeta': [], + }); + await withMockedFetch(() => new Response(body, { headers: { 'content-type': 'application/json' } }), async () => { + const executed = operation === 'openaiChatCompletions' + ? run(pipelines.openaiChatCompletions!, input, services) + : run(pipelines.anthropicMessagesCountTokens!, input, services); + await expect(executed).rejects.toBe(error); + expect(getFailureFacts(error)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'wire-model' }); + }); + }); +} diff --git a/packages/provider/src/stages/decode-response.ts b/packages/provider/src/stages/decode-response.ts index f33f8b397e..1ffd9f4994 100644 --- a/packages/provider/src/stages/decode-response.ts +++ b/packages/provider/src/stages/decode-response.ts @@ -36,8 +36,9 @@ export const decodeProviderResponse = (operatio const output = (answer: ProviderOperationOutputs[O] | ProviderProtocolFailure | null) => move({ ...back, 'response.provider.output': answer }); if (exchange.type === 'transportFailure' || exchange.status < 200 || exchange.status >= 300) return output(null); if ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && back['response.provider.responsesAction'] === 'compact' || operation === 'anthropicMessagesCountTokens') { - const text = await exchangeResponse(exchange).text(); + const response = exchangeResponse(exchange); if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const text = await response.text(); let body: unknown; try { body = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; @@ -47,8 +48,9 @@ export const decodeProviderResponse = (operatio } const contentType = exchange.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1]; if (exchange.body === null || !isEventStreamMediaType(contentType)) { - const text = await exchangeResponse(exchange).text(); + const response = exchangeResponse(exchange); if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const text = await response.text(); return output(protocolFailure(exchange, text)); } const signal = use.httpCall(facts['request.http.callId']).signal; From f5d0a01efb33a0181dd5d4db62a30834c5be54dd Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 08:44:24 +0800 Subject: [PATCH 141/156] fix(provider): transfer finite bodies before native reads Release ownership to the native finite-body consumer before text or arrayBuffer reads begin. Preserve original read errors and known endpoint facts without adding cancellation failures from a locked body. Actual Codex401 and Claude200/403 operation chains cover the transfer. --- .../provider-claude-code/__tests__/provider_test.ts | 12 ++++++++++++ .../src/stages/decode-response.ts | 2 +- .../src/stages/observe-response.ts | 2 +- packages/provider-codex/__tests__/provider_test.ts | 13 ++++++++++++- .../provider-codex/src/stages/observe-response.ts | 2 +- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/packages/provider-claude-code/__tests__/provider_test.ts b/packages/provider-claude-code/__tests__/provider_test.ts index fb44d7a94c..1ab2199914 100644 --- a/packages/provider-claude-code/__tests__/provider_test.ts +++ b/packages/provider-claude-code/__tests__/provider_test.ts @@ -4,6 +4,7 @@ import { buildClaudeCodeCatalog, type ClaudeCodeApiModel } from '../src/models.t import { pricingForClaudeCodeModelKey } from '../src/pricing.ts'; import { createClaudeCodeProvider } from '../src/provider.ts'; import type { ClaudeCodeAccessTokenEntry, ClaudeCodeAccountCredential, ClaudeCodeUpstreamState } from '../src/state.ts'; +import { getFailureFacts, type Event } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; import { initProviderRepo, type FlagId, type AnthropicMessagesUpstreamCallOptions, type UpstreamRecord } from '@floway-dev/provider'; import { collectChatProviderPipeline, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, readJsonRequest } from '@floway-dev/test-utils'; @@ -214,6 +215,17 @@ describe('createClaudeCodeProvider — Messages operation stages', () => { expect(bodies.every(body => JSON.stringify(body.messages[0]).includes('Shared system instructions'))).toBe(true); }); + test.each([200, 403])('finite body readers preserve an original stream failure at status%s', async status => { + const original = new Error('Claude body broke'); + const body = new ReadableStream({ start(controller) { controller.error(original); } }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(body, { status, headers: { 'content-type': 'application/json' } })); + const events: Event[] = []; + const caught = await collectChatProviderPipeline(createClaudeCodeProvider(currentRecord), 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [] }, undefined, noopAnthropicMessagesUpstreamCallOptions(), { dump: event => { events.push(event); } }).catch((error: unknown) => error); + expect(caught).toBe(original); + expect(getFailureFacts(caught)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'claude-sonnet-4-5-20250929' }); + expect(events.find(event => event.type === 'stage.failed')).toMatchObject({ error: original }); + }); + test('observing a JSON error preserves its original UTF8 BOM bytes', async () => { const body = { error: { type: 'permission_error', message: 'A beta feature is unavailable' } }; const bytes = new TextEncoder().encode(`\uFEFF${JSON.stringify(body)}`); diff --git a/packages/provider-claude-code/src/stages/decode-response.ts b/packages/provider-claude-code/src/stages/decode-response.ts index fca2a1df6b..593a7c643c 100644 --- a/packages/provider-claude-code/src/stages/decode-response.ts +++ b/packages/provider-claude-code/src/stages/decode-response.ts @@ -18,8 +18,8 @@ export const decodeClaudeCodeResponse = (upstreamId: string) => defineStage= 300) return move({ ...back, 'response.provider.output': null }); const response = exchangeResponse(exchange); if (exchange.body === null || !isEventStreamMediaType(response.headers.get('content-type'))) { - const text = await response.text(); if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const text = await response.text(); let body: unknown; try { body = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; body = text; } const snippet = text.length === 0 ? '' : text.length > 1024 ? `${text.slice(0, 1024)}...[truncated]` : text; diff --git a/packages/provider-claude-code/src/stages/observe-response.ts b/packages/provider-claude-code/src/stages/observe-response.ts index 7d117e6ea4..4d5632e9b2 100644 --- a/packages/provider-claude-code/src/stages/observe-response.ts +++ b/packages/provider-claude-code/src/stages/observe-response.ts @@ -26,9 +26,9 @@ export const observeClaudeCodeResponse = (upstreamId: string) => defineStage {}); const bytes = await response.arrayBuffer(); const text = new TextDecoder().decode(bytes); - if (exchange.body !== null) setRelease(exchange.body, async () => {}); try { failureBody = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; failureBody = text; } const terminal = detectTerminalSentinel(response.status, text); if (terminal !== null) tasks.push(persistTerminalAccountState(upstreamId, terminal, response.status === 400 ? 'org_disabled_400_sentinel' : 'org_banned_403_sentinel', response.status, use.log.warn)); diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index 69fe979640..e2c31854f8 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -5,7 +5,7 @@ import { CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY, CODEX_RESPONSES_LITE_HEADER } import { createCodexProvider } from '../src/provider.ts'; import type { CodexAccessTokenEntry, CodexUpstreamState } from '../src/state.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; -import { move, run, setRelease, type Event } from '@floway-dev/pipeline'; +import { getFailureFacts, move, run, setRelease, type Event } from '@floway-dev/pipeline'; import { directFetcher, initProviderRepo, providerModelFacts, type ProviderOperationPayloads, type UpstreamRecord } from '@floway-dev/provider'; import { callProviderPipeline, collectChatProviderPipeline, noopUpstreamCallOptions, readJsonRequest, stubProviderModel } from '@floway-dev/test-utils'; @@ -93,6 +93,17 @@ const oauthTokenResponse = (overrides: Partial<{ access_token: string; refresh_t }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); describe('createCodexProvider', () => { + test('the401 finite observer preserves an original stream failure without canceling its locked body', async () => { + const original = new Error('Codex refusal body broke'); + const body = new ReadableStream({ start(controller) { controller.error(original); } }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(body, { status: 401 })); + const events: Event[] = []; + const caught = await collectChatProviderPipeline(createCodexProvider(current!), 'openaiResponses', stubProviderModel({ id: 'gpt-5.4', endpoints: { openaiResponses: {} } }), { input: [] }, undefined, noopUpstreamCallOptions(), { dump: event => { events.push(event); } }).catch((error: unknown) => error); + expect(caught).toBe(original); + expect(getFailureFacts(caught)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'gpt-5.4' }); + expect(events.find(event => event.type === 'stage.failed')).toMatchObject({ error: original }); + }); + test('access-only401 remains readable while terminal persistence failure reaches the run outcome', async () => { current = accessOnlyRecord(freshAccessToken); const writeError = new Error('terminal state write failed'); diff --git a/packages/provider-codex/src/stages/observe-response.ts b/packages/provider-codex/src/stages/observe-response.ts index cebb8f44f4..799d3a6201 100644 --- a/packages/provider-codex/src/stages/observe-response.ts +++ b/packages/provider-codex/src/stages/observe-response.ts @@ -22,8 +22,8 @@ export const observeCodexResponse = (config: CodexPipelineConfig, operation: Cod let classified: Response; let failureBody: unknown = null; if (exchange.status === 401) { - const parsed = decodeCodexUpstreamError(await response.text()); if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const parsed = decodeCodexUpstreamError(await response.text()); failureBody = parsed.body; if (call.account.refresh_token === null) pending.push(call.effects.persistTerminalState('session_terminated', parsed.message)); classified = await classifyCodexUnauthorizedResponse(call, response, parsed); From 27a247964ad81475f5569d525488946dfd4dc773 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 08:47:16 +0800 Subject: [PATCH 142/156] Propagate hosted subrequest preparation failures through their run Preserve internal exceptions, failure facts and stage markers instead of turning preparation faults into upstream refusals. Child diagnostics use status 500; the image tool boundary keeps its existing native terminal failure. Fixtures close both independent recordings before awaiting durable work. --- .../hosted-tools/image-sub-request_test.ts | 31 +++++++++++++++++-- .../hosted-tools/search-sub-request_test.ts | 8 +++-- .../hosted-tools/image-generation.ts | 15 ++++++--- .../hosted-tools/image-sub-request.ts | 2 +- .../hosted-tools/image-sub-request/prepare.ts | 16 ++-------- .../hosted-tools/search-sub-request.ts | 2 +- 6 files changed, 47 insertions(+), 27 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts index a284630668..ad5f46cfba 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts @@ -1,5 +1,6 @@ import { afterEach, expect, test, vi } from 'vitest'; +import * as imageRequest from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts'; import type { ImageGenerationRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts'; import { runImageGenerationSubRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts'; import * as resolution from '../../../../../src/data-plane/providers/resolution.ts'; @@ -12,7 +13,7 @@ import { InMemoryRepo } from '../../../../repo/memory.ts'; import { flushBackground, trackBackground } from '../../../../test-utils/background-tracker.ts'; import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; import { stubProviderPipeline } from '../../../../test-utils/provider-pipeline.ts'; -import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; +import { createRunReader, getFailureFacts, type DumpEvent } from '@floway-dev/pipeline'; import { stubModelCandidate } from '@floway-dev/test-utils'; const apiKey: ApiKey = { id: 'image-key', userId: 1, name: 'Image key', key: 'floway-image-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; @@ -54,12 +55,14 @@ for (const action of ['generate', 'edit'] as const) for (const consume of [false await run.lifecycle.return(undefined as never); } await run.drain(); + dump?.finalize(200, 0); await flushBackground(); expect(generate).toHaveBeenCalledTimes(action === 'generate' ? 1 : 0); expect(edit).toHaveBeenCalledTimes(action === 'edit' ? 1 : 0); expect(parent.attempt.telemetry).toBeUndefined(); - expect(dumps.stored).toHaveLength(1); - const record = dumps.stored[0]!.record; + expect(dumps.stored).toHaveLength(2); + expect(dumps.stored.filter(item => item.record.meta.path === '/v1/responses')).toHaveLength(1); + const record = dumps.stored.find(item => item.record.meta.path !== '/v1/responses')!.record; expect(record.meta.path).toBe(action === 'edit' ? '/images/edits' : '/images/generations'); const read = createRunReader(); const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); @@ -76,3 +79,25 @@ for (const action of ['generate', 'edit'] as const) for (const consume of [false expect(observations).toMatchObject([{ model: 'image-model', modelKey: 'backend-image', requests: 1 }]); }); } + +test('hosted image preparation preserves an internal TypeError and records the failed child boundary', async () => { + initRepo(new InMemoryRepo()); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); + const fault = new TypeError('image preparation state is invalid'); + vi.spyOn(imageRequest, 'prepareImageRequest').mockRejectedValue(fault); + const request: ImageGenerationRequest = { prompt: 'tree', action: 'generate', config: { model: 'image-model', action: 'generate' }, sources: [] }; + await expect(runImageGenerationSubRequest(parent, request)).rejects.toBe(fault); + expect(getFailureFacts(fault)?.['request.imageGeneration.canonical']).toBe(request); + dump?.finalize(200, 0); + await flushBackground(); + expect(dumps.stored).toHaveLength(2); + const record = dumps.stored.find(item => item.record.meta.path === '/images/generations')!.record; + expect(record.meta.status).toBe(500); + expect(record.meta.error).toEqual({ kind: 'failed', reason: fault.message }); + const events = eventsOf(record); + const entered = events.find(event => event.type === 'stage.entered' && event.name === 'prepareHostedImageGeneration'); + expect(entered).toBeDefined(); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.failed', stageId: entered!.stageId })); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts index 0a7cb5fb8a..2e8e019585 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts @@ -43,11 +43,13 @@ for (const failed of [false, true]) { if (failed) await expect(invocation).rejects.toBe(fault); else expect(await invocation).toMatchObject({ outputText: 'documentation' }); expect(dispatched).toBe(true); + dump?.finalize(200, 0); await flushBackground(); - expect(dumps.stored).toHaveLength(1); - const record = dumps.stored[0]!.record; + expect(dumps.stored).toHaveLength(2); + expect(dumps.stored.filter(item => item.record.meta.path === '/v1/responses')).toHaveLength(1); + const record = dumps.stored.find(item => item.record.meta.path !== '/v1/responses')!.record; expect(record.meta.path).toBe('/alpha/search'); - expect(record.meta.status).toBe(failed ? 502 : 200); + expect(record.meta.status).toBe(failed ? 500 : 200); const read = createRunReader(); const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); const entered = decoded.find(event => event?.facts && 'request.webSearch.canonical' in event.facts); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts index 5418086470..66b9bf0b96 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts @@ -1,5 +1,5 @@ import { portableImageRequest, type ImageBackendConfig } from './image-sub-request/request.ts'; -import { imageTerminal, isRetryableImageError } from './image-sub-request/result.ts'; +import { imageTerminal, isRetryableImageError, serverError } from './image-sub-request/result.ts'; import { runImageGenerationSubRequest } from './image-sub-request.ts'; import type { HostedToolLifecycleEvent, HostedToolRegistration, HostedToolTerminal } from './types.ts'; import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; @@ -957,10 +957,15 @@ export const imageGenerationHostedTool: HostedToolRegistration = async (invocati // Its own run: its own prologue, its own settlement, its own record. What this // yields is that run's lifecycle, spliced into the turn that asked for it. run: () => (async function* (): AsyncGenerator { - const call = await runImageGenerationSubRequest( - state.gateway, - portableImageRequest(promptArg, operation.action, state.config, sources), - ); + let call; + try { + call = await runImageGenerationSubRequest( + state.gateway, + portableImageRequest(promptArg, operation.action, state.config, sources), + ); + } catch (error) { + return imageTerminal(promptArg, operation.action, { ok: false, error: serverError(error) }); + } try { return yield* call.lifecycle; } finally { diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts index 33e1e25655..20e38e9254 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts @@ -56,7 +56,7 @@ export const runImageGenerationSubRequest = async ( }; } catch (error) { dump?.failed(error); - dump?.finalize(502, 0); + dump?.finalize(500, 0); throw error; } }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts index 30865297e5..dfda8f9a4f 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts @@ -9,7 +9,6 @@ export const prepareHostedImageGeneration = defineStage< Fields<'request.imageGeneration.canonical' | 'request.openaiImages.canonical' | 'serve.model' | 'ingress.openaiImages.wantsStream'>, Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, - Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, GatewayServices >({ name: 'prepareHostedImageGeneration', @@ -17,19 +16,8 @@ export const prepareHostedImageGeneration = defineStage< request: { needs: ['request.imageGeneration.canonical'], consumes: [], provides: ['request.openaiImages.canonical', 'serve.model', 'ingress.openaiImages.wantsStream'] }, response: { needs: [], consumes: [], provides: [] }, }, - return: { provides: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable'] }, - execute: async (facts, next, use) => { - let request; - try { - request = await prepareImageRequest(facts['request.imageGeneration.canonical']); - } catch (error) { - use.gateway.dump?.failed(error); - return move({ - ...facts, 'response.openaiImages.canonical': { status: 502, message: error instanceof Error ? error.message : String(error) }, - 'response.openaiImages.streamedUsage': null, 'response.http.status': 502, 'response.http.headers': [], - 'response.http.body': null, 'response.usage.billable': [], - }); - } + execute: async (facts, next) => { + const request = await prepareImageRequest(facts['request.imageGeneration.canonical']); return await next({ ...facts, 'request.openaiImages.canonical': move(request), 'serve.model': facts['request.imageGeneration.canonical'].config.model, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts index 4302ed0008..71c173aac4 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts @@ -40,7 +40,7 @@ export const runWebSearchSubRequest = async ( return facts['response.webSearch.ir']; } catch (error) { dump?.failed(error); - dump?.finalize(502, 0); + dump?.finalize(500, 0); throw error; } }; From a0ffb9ea8174caeecb63250cd93e74db004ba807 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 08:47:35 +0800 Subject: [PATCH 143/156] Exclude local handshakes from billable AI calls Warmup and model probes retain diagnostic model attribution while providing no billable entities because no upstream was called. Preserve their local status and prove HTTP/WS handshakes record no usage or latency before one real continuation call. --- .../chat/anthropic-messages/http_test.ts | 7 +----- .../answer-websocket-warmup_test.ts | 6 ++--- .../chat/openai-responses/websocket_test.ts | 12 +++++++++- .../answer-claude-code-probe.ts | 23 ++++++------------- .../answer-websocket-warmup.ts | 9 ++++---- 5 files changed, 27 insertions(+), 30 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index 0108af05d2..ec2a1e1fb4 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -191,13 +191,8 @@ test('POST /v1/messages answers the Claude Code model-validation probe without c assertEquals(body.usage.output_tokens, 0); assertEquals(callAnthropicMessages.mock.calls.length, 0); - // The turn is recorded as served, at zero cost, and contributes no latency - // sample — there was no upstream call to measure. await flushBackground(); - const usage = await repo.usage.listAll(); - assertEquals(usage.length, 1); - assertEquals(usage[0]?.requests, 1); - assertEquals(usage[0]?.metrics, []); + assertEquals(await repo.usage.listAll(), []); assertEquals(await repo.performance.listAll(), []); }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts index de8bf7f280..3ac37781aa 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts @@ -14,10 +14,10 @@ const exercise = async (generate: boolean | null | undefined) => { const dial = vi.fn(); const ending = defineStage, Record>({ name: 'scriptedWarmupDial', - return: { provides: ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.body'] }, + return: { provides: ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, execute: async facts => { dial(); - return move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream', frames: (async function* () { yield doneFrame(); })() }, 'response.chat.openaiResponses.streamedUsage': null, 'response.usage.billable': [], 'response.http.headers': [], 'response.http.body': null }); + return move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream', frames: (async function* () { yield doneFrame(); })() }, 'response.chat.openaiResponses.streamedUsage': null, 'response.usage.billable': [], 'response.http.headers': [], 'response.http.status': 200, 'response.http.body': null }); }, }); const payload: CanonicalOpenAIResponsesPayload = { model: 'client-model', input: [{ type: 'message', role: 'developer', content: 'Base instructions' }], ...(generate === undefined ? {} : { generate }) }; @@ -41,7 +41,7 @@ test('a prewarm completes locally without dispatch timing and preserves its requ expect(terminal).toMatchObject({ type: 'event', event: { response: { model: candidate.model.id, output: [], status: 'completed', usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 } } } }); expect(gateway.attempt.telemetry).toBeUndefined(); expect(gateway.attempt.timing.upstreamCallStartedAt).toBeNull(); - expect(facts['response.usage.billable']).toEqual([expect.objectContaining({ quantities: {} })]); + expect(facts['response.usage.billable']).toEqual([]); }); for (const generate of [undefined, null, true]) { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index e4254931d8..aa5bad6e41 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -935,7 +935,9 @@ test('OpenAI Responses WebSocket store:false keeps session snapshots without dur }); test('OpenAI Responses WebSocket answers a Codex generate:false prewarm locally and continues from it', async () => { - const { apiKey } = await setupAppTest(); + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); const upstreamBodies: Record[] = []; await withMockedFetch( @@ -988,6 +990,13 @@ test('OpenAI Responses WebSocket answers a Codex generate:false prewarm locally // routed turn states. assertEquals(prewarmCompleted.response?.output?.map(item => item.type), ['reasoning']); assertEquals(upstreamBodies.length, 0); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + assertEquals(await repo.usage.listAll(), []); + assertEquals(await repo.performance.listAll(), []); + assertEquals(dumps.stored[0]?.record.meta.model, 'gpt-direct-responses'); + assertExists(dumps.stored[0]?.record.meta.upstream); + assertEquals(dumps.stored[0]?.record.meta.inputTokens, null); + assertEquals(dumps.stored[0]?.record.meta.outputTokens, null); const turnTerminal = waitForMessages(client, messages => messages.some(isTerminalResponseEvent)); client.send(JSON.stringify({ @@ -1002,6 +1011,7 @@ test('OpenAI Responses WebSocket answers a Codex generate:false prewarm locally await turnTerminal; assertEquals(upstreamBodies.length, 1); + await vi.waitFor(async () => assertEquals((await repo.usage.listAll()).reduce((sum, row) => sum + row.requests, 0), 1)); const body = upstreamBodies[0] as { generate?: unknown; previous_response_id?: unknown; input: Array<{ type: string; role?: string; content?: unknown }> }; assertEquals(Object.hasOwn(body, 'generate'), false); assertEquals(body.previous_response_id, undefined); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts index 99b9e0d0f6..7cbc1b9615 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts @@ -5,11 +5,6 @@ import { telemetryModelIdentity } from '../../shared/telemetry/attribution.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { providerModelOf } from '@floway-dev/provider'; -/** - * The native wire. It dials Anthropic Messages and provides the answer at this family's own response - * key — which is what will make it interchangeable with a translated chain: both hand up - * `response.chat.anthropicMessages`, and the stage above cannot tell which ran. - */ /** * Answers Claude Code's one-token probe itself, rather than spending an upstream turn on it. * @@ -28,11 +23,11 @@ export const answerClaudeCodeProbe = defineStage< Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers'>, Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers'>, Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' - | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' - | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' - | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, ChatServices >({ name: 'answerClaudeCodeProbe', @@ -53,7 +48,7 @@ export const answerClaudeCodeProbe = defineStage< 'response.chat.anthropicMessages', 'response.chat.anthropicMessages.streamedUsage', 'response.usage.billable', - 'response.http.headers', 'response.http.body', + 'response.http.headers', 'response.http.body', 'response.http.status', ], }, execute: async (facts, next, use) => { @@ -64,6 +59,7 @@ export const answerClaudeCodeProbe = defineStage< // The probe is answered *for* this candidate, so it is the one a follow-up turn carrying // our own state must come back to — the same statement a dialled attempt makes. use.selectAffinity(candidate); + use.gateway.dump?.success(telemetryModelIdentity(candidate, providerModelOf(candidate).id), null); await use.log.debug('answering a Claude Code probe without dialling', { upstream: facts['route.attempt'].upstreamId }); return move({ ...facts, @@ -71,15 +67,10 @@ export const answerClaudeCodeProbe = defineStage< kind: 'stream' as const, frames: probeFrames(facts['request.chat.anthropicMessages'].model), }, - // Nothing streams from an upstream here, so there is nothing still to read: the row is - // written now, at zero, which keeps the request visible without inventing latency. 'response.chat.anthropicMessages.streamedUsage': null, - 'response.usage.billable': [{ - identity: telemetryModelIdentity(candidate, providerModelOf(candidate).id), - quantities: {}, - }], + 'response.usage.billable': [], 'response.http.headers': [], - 'response.http.body': null, + 'response.http.body': null, 'response.http.status': 200, }); }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts b/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts index 02f4bb41b3..5ae1dbb181 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts @@ -7,8 +7,8 @@ import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult } from '@fl import { providerModelOf } from '@floway-dev/provider'; type Request = Pick; -type Response = Pick; -const provides = ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.body'] as const; +type Response = Pick; +const provides = ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status'] as const; // Codex continues from the response id of its WebSocket prewarm. The source chain has // already staged its input; normal client egress commits that snapshot without inference. @@ -25,6 +25,7 @@ export const answerOpenAIResponsesWebSocketWarmup = defineStage Date: Thu, 1 Oct 2026 08:49:06 +0800 Subject: [PATCH 144/156] Encode chat HTTP replies inside settlement Add the shared client JSON codec above every HTTP emitter while keeping canonical response objects in facts and streaming frames unchanged. Require response status from native calls, local answers and domain refusals instead of inventing success defaults. Verify original JSON serialization failures retain measured usage and count as failed requests. --- .../anthropic-messages/web-search-drive.ts | 4 +- .../data-plane/chat/client-json_test.ts | 50 +++++++++++++++++++ .../data-plane/chat/client-streams_test.ts | 4 +- .../chat/openai-responses/emit_test.ts | 6 +-- .../openai-responses/hosted-tools/drive.ts | 4 +- .../chat/resolve-candidates_test.ts | 4 +- .../chat/anthropic-messages/count-tokens.ts | 10 ++-- .../anthropic-messages/count-tokens/emit.ts | 6 +-- .../anthropic-messages/count-tokens/facts.ts | 2 +- .../count-tokens/prepare-web-search.ts | 6 +-- .../chat/anthropic-messages/emit.ts | 10 ++-- .../chat/anthropic-messages/facts.ts | 2 +- .../chat/anthropic-messages/http.ts | 4 +- .../chat/anthropic-messages/pipeline.ts | 14 +++--- .../anthropic-messages/web-search-tool.ts | 6 +-- .../gateway/src/data-plane/chat/dial-wire.ts | 2 +- .../gemini-generate-content/count-tokens.ts | 6 ++- .../count-tokens/emit.ts | 6 +-- .../count-tokens/facts.ts | 2 +- .../measure-via-anthropic-messages.ts | 6 +-- .../chat/gemini-generate-content/emit.ts | 10 ++-- .../chat/gemini-generate-content/facts.ts | 2 +- .../chat/gemini-generate-content/http.ts | 4 +- .../chat/gemini-generate-content/pipeline.ts | 8 +-- .../gateway/src/data-plane/chat/handoff.ts | 4 +- .../chat/openai-chat-completions/emit.ts | 10 ++-- .../chat/openai-chat-completions/facts.ts | 2 +- .../chat/openai-chat-completions/http.ts | 2 +- .../chat/openai-chat-completions/pipeline.ts | 14 +++--- .../chat/openai-responses/compact.ts | 2 + .../chat/openai-responses/compact/dial.ts | 2 +- .../chat/openai-responses/compact/emit.ts | 6 +-- .../chat/openai-responses/compact/facts.ts | 2 +- .../data-plane/chat/openai-responses/emit.ts | 12 ++--- .../data-plane/chat/openai-responses/facts.ts | 2 +- .../chat/openai-responses/hosted-tools.ts | 6 +-- .../data-plane/chat/openai-responses/http.ts | 4 +- .../openai-responses/hydrate-stored-items.ts | 8 +-- .../chat/openai-responses/pipeline.ts | 14 +++--- .../openai-responses/project-websocket.ts | 2 +- .../src/data-plane/chat/resolve-candidates.ts | 10 ++-- 41 files changed, 172 insertions(+), 108 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/client-json_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts index 302b05de2b..d4878cacb1 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts @@ -46,7 +46,7 @@ export const driveWebSearchStage = async ( ): Promise => { const ending = defineStage, Record>({ name: 'scriptedDial', - return: { provides: [ANSWER, 'response.usage.billable', 'response.http.headers'] }, + return: { provides: [ANSWER, 'response.usage.billable', 'response.http.headers', 'response.http.status'] }, execute: async facts => { invocation.payload = facts['request.chat.anthropicMessages'] as AnthropicMessagesPayload; const result = await dial(); @@ -56,7 +56,7 @@ export const driveWebSearchStage = async ( ? { kind: 'stream' as const, frames: result.events } : failureOf(result), 'response.usage.billable': [], - 'response.http.headers': [], + 'response.http.headers': [], 'response.http.status': result.type === 'events' ? 200 : result.status, }); }, }); diff --git a/packages/gateway/__tests__/data-plane/chat/client-json_test.ts b/packages/gateway/__tests__/data-plane/chat/client-json_test.ts new file mode 100644 index 0000000000..ec7a4a86ff --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/client-json_test.ts @@ -0,0 +1,50 @@ +import { Hono } from 'hono'; +import { expect, test } from 'vitest'; + +import { emitOpenAIChatCompletions } from '../../../src/data-plane/chat/openai-chat-completions/emit.ts'; +import { serializeClientJson } from '../../../src/data-plane/pipeline/serialize-client-json.ts'; +import { prologueFor, serveThrough } from '../../../src/data-plane/pipeline/serve.ts'; +import { writeSettlement } from '../../../src/data-plane/pipeline/settlement.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../repo/memory.ts'; +import { flushBackground, trackBackground } from '../../test-utils/background-tracker.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move } from '@floway-dev/pipeline'; +import { testTelemetryModelIdentity } from '@floway-dev/test-utils'; + +test('a client JSON formatting fault is inside the run settlement boundary', async () => { + const repo = new InMemoryRepo(); + initRepo(repo); + const now = performance.now(); + const gateway = mockChatGatewayCtx({ + backgroundScheduler: trackBackground, attempt: { + timing: { upstreamCallStartedAt: now - 10, firstOutputTokenAt: now - 5 }, + telemetry: { keyId: 'json-key', model: 'model', upstream: 'upstream', operation: 'chat', runtimeLocation: 'TEST' }, + }, + }); + const ending = defineStage, Record>({ + name: 'scriptedValue', + return: { provides: ['response.chat.openaiChatCompletions', 'response.chat.openaiChatCompletions.streamedUsage', 'response.http.headers', 'response.http.status', 'response.usage.billable'] }, + execute: async facts => move({ + ...facts, + 'response.chat.openaiChatCompletions': { kind: 'value', body: { id: 'completion', extension: 1n } }, + 'response.chat.openaiChatCompletions.streamedUsage': null, + 'response.http.headers': [], 'response.http.status': 200, + 'response.usage.billable': [{ identity: testTelemetryModelIdentity, quantities: { input_tokens: '3', output_tokens: '2' } }], + }), + }); + const pipeline = compose, Record & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] }>('clientJson', [ + writeSettlement(facts => Number(facts['response.http.status']) >= 400, 'response.chat.openaiChatCompletions.streamedUsage'), + serializeClientJson('response.chat.openaiChatCompletions.rendered'), emitOpenAIChatCompletions, ending, + ]); + const app = new Hono(); + app.post('/v1/chat/completions', c => serveThrough(c, prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }), pipeline, + move({ 'ingress.chat.openaiChatCompletions.wantsStream': false, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false }), + facts => ({ body: facts['response.http.jsonBody'] as Uint8Array, contentType: 'application/json' }))); + const response = await app.request('/v1/chat/completions', { method: 'POST' }); + expect(response.status).toBe(500); + expect(await response.json()).toMatchObject({ error: { name: 'TypeError', message: 'Do not know how to serialize a BigInt' } }); + await flushBackground(); + expect(await repo.usage.listAll()).toMatchObject([{ requests: 1 }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, errorsWithOutput: 1, errorsNoOutput: 0 }]); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts b/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts index 476f65f50f..876f188f52 100644 --- a/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts @@ -64,9 +64,9 @@ for (const fixture of cases) { const dump = openRunDump(key, { method: 'POST', path: '/v1/chat', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); if (dump === null) throw new Error('retained client request has no run recording'); const dial = defineStage, Record>({ - name: 'scriptedDial', return: { provides: [fixture.response, 'response.http.headers', `${fixture.response}.streamedUsage`] }, + name: 'scriptedDial', return: { provides: [fixture.response, 'response.http.headers', 'response.http.status', `${fixture.response}.streamedUsage`] }, execute: async facts => move({ - ...facts, 'response.http.headers': [], [`${fixture.response}.streamedUsage`]: null, + ...facts, 'response.http.headers': [], 'response.http.status': 200, [`${fixture.response}.streamedUsage`]: null, [fixture.response]: { kind: 'stream', frames: { async *[Symbol.asyncIterator]() { yield* fixture.upstream; } } }, }), }); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts index 62a44f0bab..a10d98d12e 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts @@ -23,7 +23,7 @@ const key: ApiKey = { }; type Entry = Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>; -type Answer = Entry & Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'>; +type Answer = Entry & Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>; type Exit = Fields<'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage'>; beforeEach(() => { @@ -74,10 +74,10 @@ for (const failureAt of ['before-response', 'upstream-stream', 'snapshot', 'rend })(); const dial = defineStage({ name: 'scriptedDial', - return: { provides: ['response.chat.openaiResponses', 'response.usage.billable', 'response.http.headers'] }, + return: { provides: ['response.chat.openaiResponses', 'response.usage.billable', 'response.http.headers', 'response.http.status'] }, execute: async facts => move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream', frames: upstream }, - 'response.usage.billable': [{ identity: testTelemetryModelIdentity, quantities: {} }], 'response.http.headers': [], + 'response.usage.billable': [{ identity: testTelemetryModelIdentity, quantities: {} }], 'response.http.headers': [], 'response.http.status': 200, }), }); const outcome = await run(compose('clientFailure', [ diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts index 0a6bf0913c..c0c7b85282 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts @@ -74,7 +74,7 @@ export const driveHostedToolStage = (registrations: readonly HostedToolRegistrat ): Promise | null }> => { const ending = defineStage, Record>({ name: 'scriptedDial', - return: { provides: [ANSWER, STREAMED_USAGE, 'response.usage.billable', 'response.http.headers', 'response.http.body'] }, + return: { provides: [ANSWER, STREAMED_USAGE, 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, execute: async facts => { invocation.payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; const result = await dial(); @@ -87,7 +87,7 @@ export const driveHostedToolStage = (registrations: readonly HostedToolRegistrat 'response.usage.billable': result.type === 'events' ? [{ identity: result.modelIdentity, quantities: {} }] : [], - 'response.http.headers': [], + 'response.http.headers': [], 'response.http.status': result.type === 'events' ? 200 : result.status, 'response.http.body': null, }); }, diff --git a/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts b/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts index f2cbd77158..2698d8ef75 100644 --- a/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts @@ -43,8 +43,8 @@ test('resolution publishes pure candidate request projections without provider h const dispatch = vi.fn(); const terminal = defineStage, Record>({ name: 'observeResolution', - return: { provides: ['response.usage.billable', 'response.http.headers', 'response.http.body'] }, - execute: async facts => move({ ...facts, 'response.usage.billable': [], 'response.http.headers': [], 'response.http.body': null }), + return: { provides: ['response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + execute: async facts => move({ ...facts, 'response.usage.billable': [], 'response.http.headers': [], 'response.http.status': 200, 'response.http.body': null }), }); const registry = createCandidateRegistry(); const outcome = await run(compose<{ 'serve.model': string } & Pick, Record>('resolve', [resolver, terminal]), move({ diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index 7e0f44c9d3..9cea57040f 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -1,12 +1,13 @@ -import { emitAnthropicMessagesTokenCount } from './count-tokens/emit.ts'; -import type { AnthropicMessagesCountTokensEntry, AnthropicMessagesCountTokensExit } from './count-tokens/facts.ts'; -import { isFailure } from '../../pipeline/facts.ts'; -import { writeSettlement } from '../../pipeline/settlement.ts'; import { materializeAttempt } from '../materialize-attempt.ts'; import { resolveChatCandidates } from '../resolve-candidates.ts'; +import { emitAnthropicMessagesTokenCount } from './count-tokens/emit.ts'; +import type { AnthropicMessagesCountTokensEntry, AnthropicMessagesCountTokensExit } from './count-tokens/facts.ts'; import { narrowing } from './count-tokens/target.ts'; import { anthropicMessagesCountTokensWire } from './count-tokens/wire.ts'; +import { isFailure } from '../../pipeline/facts.ts'; import { failover } from '../../pipeline/failover.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; export const anthropicMessagesCountTokensPipeline = (): Pipeline => @@ -16,6 +17,7 @@ export const anthropicMessagesCountTokensPipeline = (): Pipeline Number(handedUp['response.http.status']) >= 400, undefined, false), + serializeClientJson('response.chat.anthropicMessages.rendered'), emitAnthropicMessagesTokenCount, resolveChatCandidates(narrowing), failover({ diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts index f96f4c7cdb..2c0d17ceb3 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts @@ -13,7 +13,7 @@ import { defineStage, move } from '@floway-dev/pipeline'; export const emitAnthropicMessagesTokenCount = defineStage< Record, Record, - Counted<'response.chat.anthropicMessages'> & Fields<'response.http.headers'>, + Counted<'response.chat.anthropicMessages'> & Fields<'response.http.headers' | 'response.http.status'>, Fields<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ @@ -21,7 +21,7 @@ export const emitAnthropicMessagesTokenCount = defineStage< through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.chat.anthropicMessages', 'response.http.headers'], + needs: ['response.chat.anthropicMessages', 'response.http.headers', 'response.http.status'], consumes: ['response.chat.anthropicMessages', 'response.http.headers'], provides: ['response.chat.anthropicMessages.rendered', 'response.http.status', 'response.http.headers'], }, @@ -48,7 +48,7 @@ export const emitAnthropicMessagesTokenCount = defineStage< ...rest, 'response.http.headers': forClient, 'response.chat.anthropicMessages.rendered': move(answer.body as Record), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; }, }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/facts.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/facts.ts index 8f6111edff..e02dd697c3 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/facts.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/facts.ts @@ -15,5 +15,5 @@ export type AnthropicMessagesCountTokensEntry = Fields< >; export type AnthropicMessagesCountTokensExit = Fields< - 'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers' + 'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' >; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts index c527b0c0e1..838b5e074b 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts @@ -24,7 +24,7 @@ export const prepareAnthropicMessagesWebSearchRequest = defineStage< Fields<'request.chat.anthropicMessages'>, Counted<'response.chat.anthropicMessages'>, Counted<'response.chat.anthropicMessages'>, - Counted<'response.chat.anthropicMessages'> & Fields<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, + Counted<'response.chat.anthropicMessages'> & Fields<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, ChatServices >({ name: 'prepareAnthropicMessagesWebSearchRequest', @@ -36,7 +36,7 @@ export const prepareAnthropicMessagesWebSearchRequest = defineStage< }, response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, }, - return: { provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.body'] }, + return: { provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status'] }, execute: async (facts, next) => { if (!facts['route.attempt'].flags.includes('anthropic-messages-web-search-shim')) return await next(facts); const prepared = prepareAnthropicMessagesWebSearchShimRequest(facts['request.chat.anthropicMessages']); @@ -48,7 +48,7 @@ export const prepareAnthropicMessagesWebSearchRequest = defineStage< 'response.chat.anthropicMessages': { status: 400, message: prepared.message }, 'response.usage.billable': [], 'response.http.headers': [], - 'response.http.body': null, + 'response.http.body': null, 'response.http.status': 400, }); } // A request that engaged nothing comes back by identity, so the record shows no change diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts index 9455c1064c..5ebb95a430 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts @@ -25,7 +25,7 @@ import { toInternalDebugError } from '@floway-dev/provider'; export const emitAnthropicMessages = defineStage< Fields<'ingress.chat.anthropicMessages.wantsStream'>, Fields<'ingress.chat.anthropicMessages.wantsStream'>, - Fields<'ingress.chat.anthropicMessages.wantsStream' | 'response.chat.anthropicMessages' | 'response.http.headers' | 'response.chat.anthropicMessages.streamedUsage'>, + Fields<'ingress.chat.anthropicMessages.wantsStream' | 'response.chat.anthropicMessages' | 'response.http.headers' | 'response.http.status' | 'response.chat.anthropicMessages.streamedUsage'>, Fields<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.anthropicMessages.streamedUsage'>, ChatServices >({ @@ -33,7 +33,7 @@ export const emitAnthropicMessages = defineStage< through: { request: { needs: ['ingress.chat.anthropicMessages.wantsStream'], consumes: [], provides: [] }, response: { - needs: ['response.chat.anthropicMessages', 'response.http.headers', 'response.chat.anthropicMessages.streamedUsage'], + needs: ['response.chat.anthropicMessages', 'response.http.headers', 'response.http.status', 'response.chat.anthropicMessages.streamedUsage'], consumes: ['response.chat.anthropicMessages', 'response.http.headers'], provides: ['response.chat.clientFrames', 'response.chat.anthropicMessages.rendered', 'response.http.status', 'response.http.headers', 'response.chat.anthropicMessages.streamedUsage'], }, @@ -63,7 +63,7 @@ export const emitAnthropicMessages = defineStage< 'response.chat.clientFrames': null, 'response.http.headers': forClient, 'response.chat.anthropicMessages.rendered': move(answer.body as Record), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; } @@ -84,7 +84,7 @@ export const emitAnthropicMessages = defineStage< 'response.chat.anthropicMessages.rendered': move( await collectClientFrames(frames, collectAnthropicMessagesProtocolEventsToResult) as unknown as Record, ), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; } const completed = framedClientStream( @@ -100,7 +100,7 @@ export const emitAnthropicMessages = defineStage< 'response.http.headers': forClient, 'response.chat.anthropicMessages.rendered': move(completed.rendered), 'response.chat.anthropicMessages.streamedUsage': move(withClientVerdict(back['response.chat.anthropicMessages.streamedUsage'], completed.failed)), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; }, }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/facts.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/facts.ts index e10241ba68..f03c0fa4cf 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/facts.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/facts.ts @@ -25,5 +25,5 @@ export type AnthropicMessagesServeEntry = Fields< export type AnthropicMessagesServeExit = Fields< 'response.chat.anthropicMessages.rendered' | 'response.chat.anthropicMessages.streamedUsage' - | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' >; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts index 8fa4433b28..b409edc6b1 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts @@ -93,7 +93,7 @@ export const anthropicMessagesHttp = { // Anthropic defines a `ping` event and its clients read one, so an idle connection is // held open with that rather than with a comment no client sees. if (isFrames(rendered)) return { frames: rendered, keepAlive: anthropicMessagesKeepAlive }; - return { body: JSON.stringify(rendered), contentType: 'application/json' }; + return { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; }, ); }, @@ -126,7 +126,7 @@ export const anthropicMessagesHttp = { 'serve.model': payload.model, }) as never, // A measurement is one body however the turn went, so there is never a stream to write. - facts => ({ body: JSON.stringify(facts['response.chat.anthropicMessages.rendered']), contentType: 'application/json' }), + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), ); }, }; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts index 8893ba9799..92e5d6c2ab 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -3,20 +3,22 @@ import { composeChat as compose } from '../compose.ts'; import { answerClaudeCodeProbe } from './answer-claude-code-probe.ts'; import { emitAnthropicMessages } from './emit.ts'; import { normalizeEmptyToolsForAnthropicMessages } from './normalize-empty-tools-tool-choice.ts'; -import { isFailure } from '../../pipeline/facts.ts'; -import { writeSettlement } from '../../pipeline/settlement.ts'; -import { resolveChatCandidates } from '../resolve-candidates.ts'; import { narrowing, anthropicMessagesTarget } from './target.ts'; -import { failover } from '../../pipeline/failover.ts'; -import { materializeAttempt } from '../materialize-attempt.ts'; import { runAnthropicMessagesWebSearchTool } from './web-search-tool.ts'; -import { dialChatWire } from '../dial-wire.ts'; import { anthropicMessagesWireFor } from './wires.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { failover } from '../../pipeline/failover.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { dialChatWire } from '../dial-wire.ts'; +import { materializeAttempt } from '../materialize-attempt.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; import type { Pipeline } from '@floway-dev/pipeline'; export const anthropicMessagesServePipeline = (): Pipeline => compose('anthropicMessagesServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), + serializeClientJson('response.chat.anthropicMessages.rendered'), emitAnthropicMessages, resolveChatCandidates(narrowing), failover({ diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts index d4b563b6fc..180d0a48b1 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts @@ -28,7 +28,7 @@ import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; type M = { [P in K]: ChatFacts[P] }; -type Answered = Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers' | 'response.chat.anthropicMessages.streamedUsage'> & Record; +type Answered = Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.chat.anthropicMessages.streamedUsage'> & Record; /** The gateway's own refusal, in Anthropic's words. It is written here rather than dialled, * because a tool declaration this gateway cannot execute would reach the upstream as a body this @@ -70,7 +70,7 @@ export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => de // A declaration this gateway cannot execute, and a search backend the operator has not // configured, are both answered here: neither is a body an upstream should be asked about. return: { - provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.chat.anthropicMessages.streamedUsage'], + provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status', 'response.chat.anthropicMessages.streamedUsage'], }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -93,7 +93,7 @@ export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => de 'response.chat.anthropicMessages.streamedUsage': null, 'response.usage.billable': [], 'response.http.headers': [], - 'response.http.body': null, + 'response.http.body': null, 'response.http.status': 400, }) as never; } diff --git a/packages/gateway/src/data-plane/chat/dial-wire.ts b/packages/gateway/src/data-plane/chat/dial-wire.ts index d6fb93035e..6dc22d376a 100644 --- a/packages/gateway/src/data-plane/chat/dial-wire.ts +++ b/packages/gateway/src/data-plane/chat/dial-wire.ts @@ -46,7 +46,7 @@ export const dialChatWire = (wiring: ChatWiring) => defineStage< // Nothing is read on the way back — a wire hands up this family's own keys and they ride // through — but this is where they enter the chain, so this is the stage that provides // them and the runner checks that the wire delivered. - response: { needs: [], consumes: [], provides: wiring.provides }, + response: { needs: [], consumes: [], provides: ['response.http.status', 'response.http.body', ...wiring.provides] }, }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index 174d3aa41c..149534cc29 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -1,7 +1,6 @@ import { emitGeminiGenerateContentTokenCount } from './count-tokens/emit.ts'; import type { GeminiGenerateContentCountTokensEntry, GeminiGenerateContentCountTokensExit } from './count-tokens/facts.ts'; -import { isFailure } from '../../pipeline/facts.ts'; -import { writeSettlement } from '../../pipeline/settlement.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; import { materializeAttempt } from '../materialize-attempt.ts'; import { resolveChatCandidates } from '../resolve-candidates.ts'; import { measureGeminiGenerateContentAsAnthropicMessages } from './count-tokens/measure-via-anthropic-messages.ts'; @@ -9,7 +8,9 @@ import { narrowing } from './count-tokens/target.ts'; import { stripSafetySettingsFromGeminiGenerateContent } from './strip-safety-settings.ts'; import { stripUnsupportedPartFieldsFromGeminiGenerateContent } from './strip-unsupported-part-fields.ts'; import { stripUnsupportedToolsFromGeminiGenerateContent } from './strip-unsupported-tools.ts'; +import { isFailure } from '../../pipeline/facts.ts'; import { failover } from '../../pipeline/failover.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; import { anthropicMessagesCountTokensWire } from '../anthropic-messages/count-tokens/wire.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; @@ -20,6 +21,7 @@ export const geminiGenerateContentCountTokensPipeline = (): Pipeline Number(handedUp['response.http.status']) >= 400, undefined, false), + serializeClientJson('response.chat.geminiGenerateContent.rendered'), emitGeminiGenerateContentTokenCount, resolveChatCandidates(narrowing), failover({ diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts index 8abe520534..6341d169cf 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts @@ -13,7 +13,7 @@ import { defineStage, move } from '@floway-dev/pipeline'; export const emitGeminiGenerateContentTokenCount = defineStage< Record, Record, - Counted<'response.chat.geminiGenerateContent'> & Fields<'response.http.headers'>, + Counted<'response.chat.geminiGenerateContent'> & Fields<'response.http.headers' | 'response.http.status'>, Fields<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers'>, ChatServices >({ @@ -21,7 +21,7 @@ export const emitGeminiGenerateContentTokenCount = defineStage< through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.chat.geminiGenerateContent', 'response.http.headers'], + needs: ['response.chat.geminiGenerateContent', 'response.http.headers', 'response.http.status'], consumes: ['response.chat.geminiGenerateContent', 'response.http.headers'], provides: ['response.chat.geminiGenerateContent.rendered', 'response.http.status', 'response.http.headers'], }, @@ -48,7 +48,7 @@ export const emitGeminiGenerateContentTokenCount = defineStage< ...rest, 'response.http.headers': forClient, 'response.chat.geminiGenerateContent.rendered': move(answer.body as Record), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; }, }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/facts.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/facts.ts index c8ca01aba7..b6df90a4fa 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/facts.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/facts.ts @@ -10,5 +10,5 @@ export type GeminiGenerateContentCountTokensEntry = Fields< >; export type GeminiGenerateContentCountTokensExit = Fields< - 'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers' + 'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' >; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts index e9c2aa895a..e20df4a70c 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts @@ -21,7 +21,7 @@ export const measureGeminiGenerateContentAsAnthropicMessages = defineStage< Fields<'request.chat.anthropicMessages'>, Counted<'response.chat.anthropicMessages'>, Counted<'response.chat.geminiGenerateContent'>, - Counted<'response.chat.geminiGenerateContent'> & Fields<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, + Counted<'response.chat.geminiGenerateContent'> & Fields<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, ChatServices >({ name: 'measureGeminiGenerateContentAsAnthropicMessages', @@ -37,7 +37,7 @@ export const measureGeminiGenerateContentAsAnthropicMessages = defineStage< provides: ['response.chat.geminiGenerateContent'], }, }, - return: { provides: ['response.chat.geminiGenerateContent', 'response.usage.billable', 'response.http.headers', 'response.http.body'] }, + return: { provides: ['response.chat.geminiGenerateContent', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status'] }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); const { 'request.chat.geminiGenerateContent': asked, ...down } = facts; @@ -62,7 +62,7 @@ export const measureGeminiGenerateContentAsAnthropicMessages = defineStage< // never opens a stream, which is why this chain carries no such key at all. 'response.usage.billable': [], 'response.http.headers': [], - 'response.http.body': null, + 'response.http.body': null, 'response.http.status': 400, }); } diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts index 73b0253f77..5c498dcac7 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts @@ -24,7 +24,7 @@ import { toInternalDebugError } from '@floway-dev/provider'; export const emitGeminiGenerateContent = defineStage< Fields<'ingress.chat.geminiGenerateContent.wantsStream'>, Fields<'ingress.chat.geminiGenerateContent.wantsStream'>, - Fields<'ingress.chat.geminiGenerateContent.wantsStream' | 'response.chat.geminiGenerateContent' | 'response.http.headers' | 'response.chat.geminiGenerateContent.streamedUsage'>, + Fields<'ingress.chat.geminiGenerateContent.wantsStream' | 'response.chat.geminiGenerateContent' | 'response.http.headers' | 'response.http.status' | 'response.chat.geminiGenerateContent.streamedUsage'>, Fields<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.geminiGenerateContent.streamedUsage'>, ChatServices >({ @@ -36,7 +36,7 @@ export const emitGeminiGenerateContent = defineStage< provides: [], }, response: { - needs: ['response.chat.geminiGenerateContent', 'response.http.headers', 'response.chat.geminiGenerateContent.streamedUsage'], + needs: ['response.chat.geminiGenerateContent', 'response.http.headers', 'response.http.status', 'response.chat.geminiGenerateContent.streamedUsage'], consumes: ['response.chat.geminiGenerateContent', 'response.http.headers'], provides: ['response.chat.clientFrames', 'response.chat.geminiGenerateContent.rendered', 'response.http.status', 'response.http.headers', 'response.chat.geminiGenerateContent.streamedUsage'], }, @@ -66,7 +66,7 @@ export const emitGeminiGenerateContent = defineStage< 'response.chat.clientFrames': null, 'response.http.headers': forClient, 'response.chat.geminiGenerateContent.rendered': move(answer.body as Record), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; } @@ -87,7 +87,7 @@ export const emitGeminiGenerateContent = defineStage< 'response.chat.geminiGenerateContent.rendered': move( await collectClientFrames(frames, collectGeminiGenerateContentProtocolEventsToResult) as unknown as Record, ), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; } const completed = framedClientStream( @@ -103,7 +103,7 @@ export const emitGeminiGenerateContent = defineStage< 'response.http.headers': forClient, 'response.chat.geminiGenerateContent.rendered': move(completed.rendered), 'response.chat.geminiGenerateContent.streamedUsage': move(withClientVerdict(back['response.chat.geminiGenerateContent.streamedUsage'], completed.failed)), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; }, }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/facts.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/facts.ts index 10be252f6b..ef7c4a0945 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/facts.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/facts.ts @@ -25,5 +25,5 @@ export type GeminiGenerateContentServeEntry = Fields< export type GeminiGenerateContentServeExit = Fields< 'response.chat.geminiGenerateContent.rendered' | 'response.chat.geminiGenerateContent.streamedUsage' - | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' >; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts index 125bb956d0..81ccd6ed3b 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts @@ -108,7 +108,7 @@ const runGeminiGenerateContentGenerate = async (c: AuthedContext, model: string, facts => { const rendered = facts['response.chat.geminiGenerateContent.rendered']; if (isFrames(rendered)) return { frames: rendered }; - return { body: JSON.stringify(rendered), contentType: 'application/json' }; + return { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; }, ); }; @@ -136,6 +136,6 @@ const runGeminiGenerateContentCountTokens = async (c: AuthedContext, model: stri 'serve.model': model, }) as never, // A measurement is one body however the turn went, so there is never a stream to write. - facts => ({ body: JSON.stringify(facts['response.chat.geminiGenerateContent.rendered']), contentType: 'application/json' }), + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), ); }; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts index 5689194b08..24d8f8ab81 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -1,9 +1,7 @@ import { STREAMED_USAGE, type GeminiGenerateContentServeEntry, type GeminiGenerateContentServeExit } from './facts.ts'; import { composeChat as compose } from '../compose.ts'; import { emitGeminiGenerateContent } from './emit.ts'; -import { suppressThoughtPartsFromGeminiGenerateContent } from './suppress-thought-parts.ts'; -import { isFailure } from '../../pipeline/facts.ts'; -import { writeSettlement } from '../../pipeline/settlement.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; import { resolveChatCandidates } from '../resolve-candidates.ts'; import { requireGeminiGenerateContentTerminal } from './require-terminal.ts'; import { narrowing, geminiGenerateContentTarget } from './target.ts'; @@ -12,13 +10,17 @@ import { materializeAttempt } from '../materialize-attempt.ts'; import { stripSafetySettingsFromGeminiGenerateContent } from './strip-safety-settings.ts'; import { stripUnsupportedPartFieldsFromGeminiGenerateContent } from './strip-unsupported-part-fields.ts'; import { stripUnsupportedToolsFromGeminiGenerateContent } from './strip-unsupported-tools.ts'; +import { suppressThoughtPartsFromGeminiGenerateContent } from './suppress-thought-parts.ts'; import { dialChatWire } from '../dial-wire.ts'; import { geminiGenerateContentWireFor } from './wires.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; import type { Pipeline } from '@floway-dev/pipeline'; export const geminiGenerateContentServePipeline = (): Pipeline => compose('geminiGenerateContentServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), + serializeClientJson('response.chat.geminiGenerateContent.rendered'), emitGeminiGenerateContent, resolveChatCandidates(narrowing), failover({ diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts index ac70a782fc..0fd82acf0c 100644 --- a/packages/gateway/src/data-plane/chat/handoff.ts +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -42,7 +42,7 @@ export interface Handoff => ({ 'response.usage.billable': [], 'response.http.headers': [], - 'response.http.body': null, + 'response.http.body': null, 'response.http.status': 400, [`${response}.streamedUsage`]: null, [response]: { status: 400, message }, }); @@ -73,7 +73,7 @@ export const handOff = , Fields<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, Fields<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk' - | 'response.chat.openaiChatCompletions' | 'response.http.headers' | 'response.chat.openaiChatCompletions.streamedUsage'>, + | 'response.chat.openaiChatCompletions' | 'response.http.headers' | 'response.http.status' | 'response.chat.openaiChatCompletions.streamedUsage'>, Fields<'response.chat.openaiChatCompletions.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.openaiChatCompletions.streamedUsage'>, ChatServices >({ @@ -35,7 +35,7 @@ export const emitOpenAIChatCompletions = defineStage< provides: [], }, response: { - needs: ['response.chat.openaiChatCompletions', 'response.http.headers', 'response.chat.openaiChatCompletions.streamedUsage'], + needs: ['response.chat.openaiChatCompletions', 'response.http.headers', 'response.http.status', 'response.chat.openaiChatCompletions.streamedUsage'], consumes: ['response.chat.openaiChatCompletions', 'response.http.headers'], provides: ['response.chat.clientFrames', 'response.chat.openaiChatCompletions.rendered', 'response.http.status', 'response.http.headers', 'response.chat.openaiChatCompletions.streamedUsage'], }, @@ -65,7 +65,7 @@ export const emitOpenAIChatCompletions = defineStage< 'response.chat.clientFrames': null, 'response.http.headers': forClient, 'response.chat.openaiChatCompletions.rendered': move(answer.body as Record), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; } @@ -86,7 +86,7 @@ export const emitOpenAIChatCompletions = defineStage< 'response.chat.openaiChatCompletions.rendered': move( await collectClientFrames(frames, collectOpenAIChatCompletionsProtocolEventsToResult) as unknown as Record, ), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; } const completed = framedClientStream>( @@ -102,7 +102,7 @@ export const emitOpenAIChatCompletions = defineStage< 'response.http.headers': forClient, 'response.chat.openaiChatCompletions.rendered': move(completed.rendered), 'response.chat.openaiChatCompletions.streamedUsage': move(withClientVerdict(back['response.chat.openaiChatCompletions.streamedUsage'], completed.failed)), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/facts.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/facts.ts index 321b360042..9a58297cc9 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/facts.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/facts.ts @@ -26,5 +26,5 @@ export type OpenAIChatCompletionsServeEntry = Fields< export type OpenAIChatCompletionsServeExit = Fields< 'response.chat.openaiChatCompletions.rendered' | 'response.chat.openaiChatCompletions.streamedUsage' - | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' >; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts index 1d8b4221dc..dd344d895b 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts @@ -66,7 +66,7 @@ export const openaiChatCompletionsHttp = { facts => { const rendered = facts['response.chat.openaiChatCompletions.rendered']; if (isFrames(rendered)) return { frames: rendered }; - return { body: JSON.stringify(rendered), contentType: 'application/json' }; + return { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; }, ); }, diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts index f09fe1e664..ef1c2a484f 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -1,20 +1,22 @@ import { STREAMED_USAGE, type OpenAIChatCompletionsServeEntry, type OpenAIChatCompletionsServeExit } from './facts.ts'; import { composeChat as compose } from '../compose.ts'; import { emitOpenAIChatCompletions } from './emit.ts'; -import { isFailure } from '../../pipeline/facts.ts'; -import { writeSettlement } from '../../pipeline/settlement.ts'; -import { resolveChatCandidates } from '../resolve-candidates.ts'; +import { normalizeEmptyToolsForOpenAIChatCompletions } from './normalize-empty-tools-tool-choice.ts'; import { narrowing, openaiChatCompletionsTarget } from './target.ts'; +import { openaiChatCompletionsWireFor } from './wires.ts'; +import { isFailure } from '../../pipeline/facts.ts'; import { failover } from '../../pipeline/failover.ts'; -import { materializeAttempt } from '../materialize-attempt.ts'; -import { normalizeEmptyToolsForOpenAIChatCompletions } from './normalize-empty-tools-tool-choice.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; import { dialChatWire } from '../dial-wire.ts'; -import { openaiChatCompletionsWireFor } from './wires.ts'; +import { materializeAttempt } from '../materialize-attempt.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; import type { Pipeline } from '@floway-dev/pipeline'; export const openaiChatCompletionsServePipeline = (): Pipeline => compose('openaiChatCompletionsServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), + serializeClientJson('response.chat.openaiChatCompletions.rendered'), emitOpenAIChatCompletions, resolveChatCandidates(narrowing), failover({ diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts index 17463c36de..8fabee2927 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -16,12 +16,14 @@ import { hostedTools } from './hosted-tools.ts'; import { hydrateStoredItems } from './hydrate-stored-items.ts'; import { openaiResponsesNarrowing, openaiResponsesTarget } from './target.ts'; import { isFailure } from '../../pipeline/facts.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; import type { Pipeline } from '@floway-dev/pipeline'; export const openaiResponsesCompactPipeline = (): Pipeline => { return compose('openaiResponsesCompact', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), + serializeClientJson('response.chat.openaiResponses.rendered'), emitOpenAIResponsesCompaction, hydrateStoredItems, resolveChatCandidates(openaiResponsesNarrowing), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts index 01c24b417d..08ebc5085f 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts @@ -41,7 +41,7 @@ export const dialOpenAIResponsesCompaction = (wires: { native: ChatWire; simulat 'response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', - 'response.http.headers', + 'response.http.headers', 'response.http.status', 'response.http.body', ], }, }, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts index 5a05329329..7d763c9540 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts @@ -26,7 +26,7 @@ import { collectOpenAIResponsesProtocolEventsToResult, type OpenAIResponsesStrea export const emitOpenAIResponsesCompaction = defineStage< Record, Record, - Compacted<'response.chat.openaiResponses'> & Fields<'response.http.headers' | 'response.chat.openaiResponses.streamedUsage'>, + Compacted<'response.chat.openaiResponses'> & Fields<'response.http.headers' | 'response.http.status' | 'response.chat.openaiResponses.streamedUsage'>, Fields<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.openaiResponses.streamedUsage' | 'response.chat.clientFrames'>, ChatServices >({ @@ -34,7 +34,7 @@ export const emitOpenAIResponsesCompaction = defineStage< through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.chat.openaiResponses', 'response.http.headers', 'response.chat.openaiResponses.streamedUsage'], + needs: ['response.chat.openaiResponses', 'response.http.headers', 'response.http.status', 'response.chat.openaiResponses.streamedUsage'], consumes: ['response.chat.openaiResponses', 'response.http.headers'], provides: [ 'response.chat.clientFrames', @@ -82,7 +82,7 @@ export const emitOpenAIResponsesCompaction = defineStage< 'response.chat.openaiResponses.rendered': move( completeOpenAIResponsesCompaction(persisted, openaiResponsesCreatedAt(use.gateway)) as unknown as Record, ), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], 'response.chat.openaiResponses.streamedUsage': move( withVerdict(rest['response.chat.openaiResponses.streamedUsage'], compactionFailed(persisted)), ), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts index 0af1e4ee01..5352495e0c 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts @@ -14,5 +14,5 @@ export type OpenAIResponsesCompactEntry = Fields< export type OpenAIResponsesCompactExit = Fields< 'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage' - | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' >; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts index 9dc020c8bd..79baa77e32 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts @@ -32,7 +32,7 @@ import { toInternalDebugError } from '@floway-dev/provider'; export const emitOpenAIResponses = (framing: OpenAIResponsesStreamFraming) => defineStage< Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>, Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>, - Fields<'ingress.chat.openaiResponses.wantsStream' | 'response.chat.openaiResponses' | 'response.http.headers' | 'response.chat.openaiResponses.streamedUsage'>, + Fields<'ingress.chat.openaiResponses.wantsStream' | 'response.chat.openaiResponses' | 'response.http.headers' | 'response.http.status' | 'response.chat.openaiResponses.streamedUsage'>, Fields<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.openaiResponses.streamedUsage'>, ChatServices >({ @@ -40,7 +40,7 @@ export const emitOpenAIResponses = (framing: OpenAIResponsesStreamFraming) => de through: { request: { needs: ['ingress.chat.openaiResponses.wantsStream', 'request.chat.openaiResponses'], consumes: [], provides: [] }, response: { - needs: ['response.chat.openaiResponses', 'response.http.headers', OPENAI_RESPONSES_STREAMED_USAGE], + needs: ['response.chat.openaiResponses', 'response.http.headers', 'response.http.status', OPENAI_RESPONSES_STREAMED_USAGE], consumes: ['response.chat.openaiResponses', 'response.http.headers'], provides: ['response.chat.clientFrames', 'response.chat.openaiResponses.rendered', 'response.http.status', 'response.http.headers', OPENAI_RESPONSES_STREAMED_USAGE], }, @@ -73,7 +73,7 @@ export const emitOpenAIResponses = (framing: OpenAIResponsesStreamFraming) => de 'response.chat.clientFrames': null, 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move(answer.body as Record), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; } @@ -101,7 +101,7 @@ export const emitOpenAIResponses = (framing: OpenAIResponsesStreamFraming) => de 'response.chat.openaiResponses.rendered': move( await collectClientFrames(frames, collectOpenAIResponsesProtocolEventsToResult) as unknown as Record, ), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], }; } catch (error) { // Nothing has gone out yet, so the fault is still a status. A turn the gateway could @@ -127,7 +127,7 @@ export const emitOpenAIResponses = (framing: OpenAIResponsesStreamFraming) => de 'response.chat.clientFrames': move(frames), 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move(frames), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], clientStream.failed)), }; } @@ -148,7 +148,7 @@ export const emitOpenAIResponses = (framing: OpenAIResponsesStreamFraming) => de 'response.chat.clientFrames': move(completed.frames), 'response.http.headers': forClient, 'response.chat.openaiResponses.rendered': move(completed.rendered), - 'response.http.status': 'response.http.status' in back ? back['response.http.status'] as number : 200, + 'response.http.status': back['response.http.status'], [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], completed.failed)), }; }, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/facts.ts index e2949ee9e9..ae7153247d 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/facts.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/facts.ts @@ -41,5 +41,5 @@ export type OpenAIResponsesServeEntry = Fields< export type OpenAIResponsesServeExit = Fields< 'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage' - | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' >; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts index 817686f745..b2d50bdc46 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts @@ -45,7 +45,7 @@ type R = { [P in K]: ChatFacts[P] }; /** What comes back through here. The streamed-usage key is the *source* family's rather than * this protocol's — the same reason a wire is told it — so the slice is open at that one key * and closed everywhere else. */ -type Answered = R<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers'> +type Answered = R<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'> & { 'response.chat.openaiResponses.streamedUsage': Deferred | null } & Record; @@ -178,7 +178,7 @@ export const hostedTools = ( // A tool declaration this gateway cannot accept is answered here rather than dialled: the // upstream would have been asked for a tool it does not implement, on a body the dispatcher wrote. return: { - provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers', 'response.http.body'], + provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status'], }, execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -201,7 +201,7 @@ export const hostedTools = ( [wiring.streamedUsage]: null, 'response.usage.billable': [], 'response.http.headers': [], - 'response.http.body': null, + 'response.http.body': null, 'response.http.status': prepared.refused.status, }) as never; } const { active } = prepared; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/http.ts b/packages/gateway/src/data-plane/chat/openai-responses/http.ts index 085d634cc0..9e40c306bf 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/http.ts @@ -95,7 +95,7 @@ export const openaiResponsesHttp = { facts => { const rendered = facts['response.chat.openaiResponses.rendered']; if (isFrames(rendered)) return { frames: rendered }; - return { body: JSON.stringify(rendered), contentType: 'application/json' }; + return { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; }, ); }, @@ -124,7 +124,7 @@ export const openaiResponsesHttp = { }) as never, // A compaction is one resource however the turn went, so there is never a stream to // write: the frames it came as were read into that resource before the run answered. - facts => ({ body: JSON.stringify(facts['response.chat.openaiResponses.rendered']), contentType: 'application/json' }), + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), ); }, }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts b/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts index e153a79348..54f0d690c6 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts @@ -14,10 +14,10 @@ const refuseStoredItems = ( status: number, message: string, extra: { readonly param: string; readonly code: string | null }, -): Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'> => ({ +): Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'> => ({ 'response.usage.billable': [], 'response.http.headers': [], - 'response.http.body': null, + 'response.http.body': null, 'response.http.status': status, 'response.chat.openaiResponses.streamedUsage': null, 'response.chat.openaiResponses': { status, @@ -47,7 +47,7 @@ export const hydrateStoredItems = defineStage< Fields<'request.chat.openaiResponses' | 'request.chat.openaiResponses.privatePayloads'>, Record, Record, - Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, + Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, ChatServices >({ name: 'hydrateStoredItems', @@ -64,7 +64,7 @@ export const hydrateStoredItems = defineStage< 'response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', - 'response.http.headers', 'response.http.body', + 'response.http.headers', 'response.http.body', 'response.http.status', ], }, execute: async (facts, next, use) => { diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 60765c5969..081f8c698d 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -9,18 +9,19 @@ import { asksForCompaction } from './compaction-policy.ts'; import { emitOpenAIResponses } from './emit.ts'; import { expandShimCompactions } from './expand-compactions.ts'; import { OPENAI_RESPONSES_STREAMED_USAGE, type OpenAIResponsesStreamFraming, type OpenAIResponsesServeEntry, type OpenAIResponsesServeExit } from './facts.ts'; +import { projectOpenAIResponsesWebSocket, type OpenAIResponsesWebSocketEntry, type OpenAIResponsesWebSocketExit } from './project-websocket.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { composeChat as compose } from '../compose.ts'; +import { dialChatWire } from '../dial-wire.ts'; +import { imageGenerationHostedTool } from './hosted-tools/image-generation.ts'; +import { webSearchHostedTool } from './hosted-tools/web-search.ts'; +import { hostedTools } from './hosted-tools.ts'; import { normalizeEmptyToolsForOpenAIResponses } from './normalize-empty-tools-tool-choice.ts'; import { summarizeForCompaction } from './summarize-for-compaction.ts'; import { openaiResponsesNarrowing, openaiResponsesTarget } from './target.ts'; import { openaiResponsesWireFor } from './wires.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; -import { composeChat as compose } from '../compose.ts'; -import { dialChatWire } from '../dial-wire.ts'; -import { imageGenerationHostedTool } from './hosted-tools/image-generation.ts'; -import { webSearchHostedTool } from './hosted-tools/web-search.ts'; -import { hostedTools } from './hosted-tools.ts'; -import { projectOpenAIResponsesWebSocket, type OpenAIResponsesWebSocketEntry, type OpenAIResponsesWebSocketExit } from './project-websocket.ts'; import type { Pipeline } from '@floway-dev/pipeline'; export const openaiResponsesServePipeline = ( @@ -30,6 +31,7 @@ export const openaiResponsesServePipeline = => { return compose('openaiResponsesServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), + ...framing === 'sse' ? [serializeClientJson('response.chat.openaiResponses.rendered')] : [], ...framing === 'events' ? [projectOpenAIResponsesWebSocket] : [], emitOpenAIResponses(framing), hydrateStoredItems, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts index 663fe5fc01..afa2a6a3f3 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts @@ -91,7 +91,7 @@ export interface OpenAIResponsesWebSocketPacket { export type OpenAIResponsesWebSocketEntry = OpenAIResponsesServeEntry & { readonly 'ingress.chat.openaiResponses.eventId': string | undefined; }; -export type OpenAIResponsesWebSocketExit = Omit & { +export type OpenAIResponsesWebSocketExit = Omit & { readonly 'response.chat.openaiResponses.websocket': AsyncIterable; }; diff --git a/packages/gateway/src/data-plane/chat/resolve-candidates.ts b/packages/gateway/src/data-plane/chat/resolve-candidates.ts index dcb4e4ccce..d0b5eaf37b 100644 --- a/packages/gateway/src/data-plane/chat/resolve-candidates.ts +++ b/packages/gateway/src/data-plane/chat/resolve-candidates.ts @@ -39,9 +39,9 @@ export interface ChatNarrowing(narrowing: ChatNarrowing) => defineStage< Slice<'serve.model'> & Pick, Slice<'serve.model' | 'serve.candidates'> & Pick & Pick, - Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, - Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'>, - Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body'> & Refusal, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'> & Refusal, ChatServices >({ name: 'resolveChatCandidates', @@ -49,7 +49,7 @@ export const resolveChatCandidates = { const model = facts['serve.model']; const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ @@ -67,7 +67,7 @@ export const resolveChatCandidates = Date: Thu, 1 Oct 2026 08:59:21 +0800 Subject: [PATCH 145/156] Compose fixed Responses transport edges Separate typed HTTP and WebSocket entry assemblies over shared model stages. Each edge now has its own declared entry/exit facts and serializer, without a generic framing default that could disagree with the actual stage array. --- .../data-plane/pipeline/assembly_test.ts | 10 ++- .../chat/openai-responses/pipeline.ts | 86 ++++++++++--------- .../chat/openai-responses/websocket.ts | 6 +- 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 6aa0d9ab7f..34a08407fd 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -13,7 +13,7 @@ import { geminiGenerateContentCountTokensPipeline } from '../../../src/data-plan import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; import { openaiResponsesCompactPipeline } from '../../../src/data-plane/chat/openai-responses/compact.ts'; -import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; +import { openaiResponsesServePipeline, openaiResponsesWebSocketPipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; import { openaiAudioTranscriptionServePipeline } from '../../../src/data-plane/openai-audio/pipeline.ts'; import { openaiCompletionsServePipeline } from '../../../src/data-plane/openai-completions/pipeline.ts'; import { openaiEmbeddingsServePipeline } from '../../../src/data-plane/openai-embeddings/pipeline.ts'; @@ -32,6 +32,7 @@ const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] ['Anthropic Messages', () => anthropicMessagesServePipeline()], ['Gemini generateContent', () => geminiGenerateContentServePipeline()], ['OpenAI Responses', () => openaiResponsesServePipeline()], + ['OpenAI Responses WebSocket', () => openaiResponsesWebSocketPipeline()], ['Anthropic Messages count_tokens', () => anthropicMessagesCountTokensPipeline()], ['Gemini generateContent countTokens', () => geminiGenerateContentCountTokensPipeline()], ['OpenAI Responses compact', () => openaiResponsesCompactPipeline()], @@ -44,3 +45,10 @@ describe('every family assembles', () => { }); } }); + +it('Responses transport entries require the facts of their actual edge', () => { + const http = openaiResponsesServePipeline(); + const websocket = openaiResponsesWebSocketPipeline(); + expect(http.entryNeeds).not.toContain('ingress.chat.openaiResponses.eventId'); + expect(websocket.entryNeeds).toContain('ingress.chat.openaiResponses.eventId'); +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts index 081f8c698d..15de8359be 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -8,7 +8,7 @@ import { projectOpenAIResponsesCollaboration } from './collaboration-shim.ts'; import { asksForCompaction } from './compaction-policy.ts'; import { emitOpenAIResponses } from './emit.ts'; import { expandShimCompactions } from './expand-compactions.ts'; -import { OPENAI_RESPONSES_STREAMED_USAGE, type OpenAIResponsesStreamFraming, type OpenAIResponsesServeEntry, type OpenAIResponsesServeExit } from './facts.ts'; +import { OPENAI_RESPONSES_STREAMED_USAGE, type OpenAIResponsesServeEntry, type OpenAIResponsesServeExit } from './facts.ts'; import { projectOpenAIResponsesWebSocket, type OpenAIResponsesWebSocketEntry, type OpenAIResponsesWebSocketExit } from './project-websocket.ts'; import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; import { composeChat as compose } from '../compose.ts'; @@ -22,45 +22,51 @@ import { openaiResponsesNarrowing, openaiResponsesTarget } from './target.ts'; import { openaiResponsesWireFor } from './wires.ts'; import { isFailure } from '../../pipeline/facts.ts'; import { writeSettlement } from '../../pipeline/settlement.ts'; -import type { Pipeline } from '@floway-dev/pipeline'; +import type { Pipeline, Stage } from '@floway-dev/pipeline'; -export const openaiResponsesServePipeline = ( - // SSE is what a run is written in when nothing else claims its frames, which is every - // entry over an HTTP body; the WebSocket transport says so because it writes its own. - framing: Framing = 'sse' as Framing, -): Pipeline => { - return compose('openaiResponsesServe', [ +export const openaiResponsesServePipeline = (): Pipeline => + compose('openaiResponsesServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), - ...framing === 'sse' ? [serializeClientJson('response.chat.openaiResponses.rendered')] : [], - ...framing === 'events' ? [projectOpenAIResponsesWebSocket] : [], - emitOpenAIResponses(framing), - hydrateStoredItems, - resolveChatCandidates(openaiResponsesNarrowing), - failover({ - failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), - owns: ['response.http.body'], - pendingUsage: OPENAI_RESPONSES_STREAMED_USAGE, - }), - materializeAttempt('request.chat.openaiResponses'), - beginStoredAttempt, - ...framing === 'events' ? [answerOpenAIResponsesWebSocketWarmup] : [], - expandShimCompactions, - projectOpenAIResponsesCollaboration, - summarizeForCompaction(asksForCompaction), - // Directly above the dial, because every descent it makes is another dial of this same - // candidate: a hosted tool the upstream does not implement is emulated by asking again with - // the tool's result folded back in, and the frames of every ask are spliced into one turn. - hostedTools([webSearchHostedTool, imageGenerationHostedTool], { - streamedUsage: OPENAI_RESPONSES_STREAMED_USAGE, - targetOf: candidate => openaiResponsesTarget.pick(candidate.model.endpoints), - }), - normalizeEmptyToolsForOpenAIResponses, - dialChatWire({ - source: 'request.chat.openaiResponses', - needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], - provides: ['response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], - pick: endpoints => openaiResponsesTarget.pick(endpoints), - wire: openaiResponsesWireFor, - }), + serializeClientJson('response.chat.openaiResponses.rendered'), + emitOpenAIResponses('sse'), + ...openaiResponsesModelStages(false), ]); -}; + +export const openaiResponsesWebSocketPipeline = (): Pipeline => + compose('openaiResponsesWebSocket', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), + projectOpenAIResponsesWebSocket, + emitOpenAIResponses('events'), + ...openaiResponsesModelStages(true), + ]); + +const openaiResponsesModelStages = (warmup: boolean): readonly Stage[] => [ + hydrateStoredItems, + resolveChatCandidates(openaiResponsesNarrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), + owns: ['response.http.body'], + pendingUsage: OPENAI_RESPONSES_STREAMED_USAGE, + }), + materializeAttempt('request.chat.openaiResponses'), + beginStoredAttempt, + ...warmup ? [answerOpenAIResponsesWebSocketWarmup] : [], + expandShimCompactions, + projectOpenAIResponsesCollaboration, + summarizeForCompaction(asksForCompaction), + // Directly above the dial, because every descent it makes is another dial of this same + // candidate: a hosted tool the upstream does not implement is emulated by asking again with + // the tool's result folded back in, and the frames of every ask are spliced into one turn. + hostedTools([webSearchHostedTool, imageGenerationHostedTool], { + streamedUsage: OPENAI_RESPONSES_STREAMED_USAGE, + targetOf: candidate => openaiResponsesTarget.pick(candidate.model.endpoints), + }), + normalizeEmptyToolsForOpenAIResponses, + dialChatWire({ + source: 'request.chat.openaiResponses', + needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => openaiResponsesTarget.pick(endpoints), + wire: openaiResponsesWireFor, + }), +]; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index c9183fd077..fa9a6d4271 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -11,7 +11,7 @@ import type { Context } from 'hono'; import { createOpenAIResponsesWsSession, type OpenAIResponsesStatefulStore } from './items/store.ts'; -import { openaiResponsesServePipeline } from './pipeline.ts'; +import { openaiResponsesWebSocketPipeline } from './pipeline.ts'; import type { OpenAIResponsesWebSocketExit } from './project-websocket.ts'; import { openRunDump } from '../../../dump/run-sink.ts'; import type { RunDump } from '../../../dump/run-sink.ts'; @@ -335,9 +335,7 @@ const handleClientMessage = async ( ctx = prologue.gateway; const { facts, drain } = await run( - // The transport frames its own answer, so the edge hands up the events rather than the - // SSE a body would have been written from. - openaiResponsesServePipeline('events'), + openaiResponsesWebSocketPipeline(), move({ 'ingress.http.headers': prologue.headers, 'ingress.chat.sourceProtocol': 'openaiResponses', From 848bf5cba2cf1a2594980467498601a34bc22568 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:03:00 +0800 Subject: [PATCH 146/156] refactor: confine provider reply fixtures to test infrastructure Remove the retired production result module and callable-model result types. The actual operation contracts remain provider-owned; fixture reply DTOs and their factory live in test-utils. Keep response cancellation as an HTTP helper for the active catalog and search callers. Moved 54 fixture imports with identical non-import syntax trees and removed stale result-layer comments. Verified 2495 affected tests, 152 follow-up provider tests, ten package typechecks, frozen lock consistency and scoped lint. Test readers adopt finite body ownership before reading and preserve cleanup error chains. --- .../anthropic-messages-count-tokens_test.ts | 4 +- .../chat/anthropic-messages-pipeline_test.ts | 4 +- .../chat/anthropic-messages/http_test.ts | 4 +- .../anthropic-messages/web-search-drive.ts | 3 +- ...mini-generate-content-count-tokens_test.ts | 4 +- .../gemini-generate-content-pipeline_test.ts | 4 +- .../chat/gemini-generate-content/http_test.ts | 4 +- .../openai-chat-completions-pipeline_test.ts | 4 +- .../chat/openai-chat-completions/http_test.ts | 4 +- .../chat/openai-responses-compact_test.ts | 4 +- .../chat/openai-responses-pipeline_test.ts | 4 +- .../openai-responses/collaboration-drive.ts | 3 +- .../collaboration-shim_test.ts | 3 +- .../openai-responses/hosted-tools/drive.ts | 3 +- .../image-generation-integration_test.ts | 4 +- .../openai-responses/hosted-tools_test.ts | 4 +- .../chat/openai-responses/http_test.ts | 4 +- .../data-plane/chat/translated-wires_test.ts | 4 +- .../data-plane/openai-audio-pipeline_test.ts | 4 +- .../openai-completions-pipeline_test.ts | 4 +- .../data-plane/rerank-status_test.ts | 4 +- .../__tests__/test-utils/provider-pipeline.ts | 3 +- .../openai-responses/response-resource.ts | 10 +- .../tools/web-search/providers/shared.ts | 2 +- packages/gateway/src/dump/types.ts | 2 +- packages/http/src/index.ts | 2 + packages/http/src/response.ts | 4 + .../stages/backfill-required-fields_test.ts | 4 +- .../hoist-user-system-to-messages_test.ts | 3 +- .../stages/inject-billing-block_test.ts | 3 +- .../stages/inject-default-template_test.ts | 3 +- .../stages/inject-identity-block_test.ts | 3 +- .../synthesize-metadata-user-id_test.ts | 3 +- .../__tests__/test-utils/call.ts | 4 +- .../inject-default-instructions_test.ts | 3 +- .../strip-unsupported-response-fields_test.ts | 3 +- .../__tests__/test-utils/call.ts | 4 +- .../apply-top-level-cache-control_test.ts | 4 +- .../compress-images_test.ts | 4 +- .../handle-speed-fast_test.ts | 4 +- .../normalize-anthropic-beta_test.ts | 3 +- .../promote-thinking-display_test.ts | 4 +- .../set-claude-agent-headers_test.ts | 4 +- .../set-compact-headers_test.ts | 4 +- .../set-initiator-header_test.ts | 4 +- .../set-interaction-id-header_test.ts | 4 +- .../set-vision-header_test.ts | 4 +- .../strip-cache-control-extensions_test.ts | 4 +- .../abort-on-tool-argument-whitespace_test.ts | 4 +- .../attach-cache-control-markers_test.ts | 4 +- .../compress-images_test.ts | 4 +- .../set-initiator-header_test.ts | 4 +- .../set-vision-header_test.ts | 4 +- .../abort-on-tool-argument-whitespace_test.ts | 3 +- .../openai-responses/compress-images_test.ts | 4 +- .../force-store-false_test.ts | 4 +- .../openai-responses/item-id-membrane_test.ts | 3 +- .../set-initiator-header_test.ts | 4 +- .../set-vision-header_test.ts | 4 +- .../set-initiator-header.ts | 5 - .../anthropic-messages/set-vision-header.ts | 8 +- packages/provider-ollama/src/fetch-models.ts | 3 +- packages/provider/src/index.ts | 22 --- packages/provider/src/provider.ts | 23 ---- packages/provider/src/result.ts | 129 ------------------ packages/test-utils/package.json | 1 + .../test-utils/src/chat-provider-pipeline.ts | 3 +- packages/test-utils/src/index.ts | 1 + packages/test-utils/src/provider-pipeline.ts | 13 +- packages/test-utils/src/provider-results.ts | 41 ++++++ pnpm-lock.yaml | 3 + 71 files changed, 150 insertions(+), 323 deletions(-) delete mode 100644 packages/provider/src/result.ts create mode 100644 packages/test-utils/src/provider-results.ts diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts index cf7f92a5c9..b4f5662a10 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts @@ -15,8 +15,8 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { AliasRules, ModelEndpoints } from '@floway-dev/protocols/common'; -import { type FlagId, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; -import type { StubChatProviderCall } from '@floway-dev/test-utils'; +import { type FlagId, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderCallResult } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts index c3e94163b6..cc519662d3 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts @@ -17,8 +17,8 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { directFetcher, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; -import type { StubChatProviderCall } from '@floway-dev/test-utils'; +import { directFetcher, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderStreamResult } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index ec2a1e1fb4..97d4a4c7e3 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -9,8 +9,8 @@ import { flushBackground } from '../../../test-utils/background-tracker.ts'; import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; +import { type ModelCandidate, directFetcher, type UpstreamCallOptions } from '@floway-dev/provider'; +import { assert, assertEquals, stubProvider, stubInternalModel, type ProviderCallResult, type ProviderStreamResult } from '@floway-dev/test-utils'; const candidatesQueue: { readonly candidates: readonly ModelCandidate[]; readonly sawModel: boolean; readonly failedUpstreams: readonly string[] }[] = []; vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts index d4878cacb1..965a3d606a 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts @@ -12,7 +12,8 @@ import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gate import { compose, defineStage, move, run } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { AnthropicMessagesInvocation, ExecuteResult, ModelCandidate } from '@floway-dev/provider'; +import type { AnthropicMessagesInvocation, ModelCandidate } from '@floway-dev/provider'; +import { type ExecuteResult } from '@floway-dev/test-utils'; const ANSWER = 'response.chat.anthropicMessages'; diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts index 1eb4e5bcc2..65e08e38aa 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts @@ -15,8 +15,8 @@ import { move, run } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -import type { AnthropicMessagesUpstreamCallOptions, ModelCandidate, ProviderCallResult } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import type { AnthropicMessagesUpstreamCallOptions, ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderCallResult } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ ...(await importOriginal()), diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts index db48a367e0..2d4549cc59 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts @@ -16,8 +16,8 @@ import { move, run } from '@floway-dev/pipeline'; import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, type GeminiGenerateContentPayload, type GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { directFetcher, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; -import type { StubChatProviderCall } from '@floway-dev/test-utils'; +import { directFetcher, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderStreamResult } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts index 049b7d6eb6..62194793b7 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts @@ -9,8 +9,8 @@ import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pip import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; +import { type ModelCandidate, directFetcher, type UpstreamCallOptions } from '@floway-dev/provider'; +import { assert, assertEquals, stubProvider, stubInternalModel, type ProviderCallResult, type ProviderStreamResult } from '@floway-dev/test-utils'; const candidatesQueue: { readonly candidates: readonly ModelCandidate[]; readonly sawModel: boolean; readonly failedUpstreams: readonly string[] }[] = []; vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts index 18cb0d2b0a..b09be27ffb 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts @@ -15,8 +15,8 @@ import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; import { type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { directFetcher, type FlagId, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; -import type { StubChatProviderCall } from '@floway-dev/test-utils'; +import { directFetcher, type FlagId, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderStreamResult } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts index a4b9898105..12d5f8b61b 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts @@ -11,8 +11,8 @@ import { flushBackground } from '../../../test-utils/background-tracker.ts'; import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ModelCandidate, directFetcher, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; +import { type ModelCandidate, directFetcher, type UpstreamCallOptions } from '@floway-dev/provider'; +import { assert, assertEquals, stubProvider, stubInternalModel, type ProviderStreamResult } from '@floway-dev/test-utils'; const candidatesQueue: { readonly candidates: readonly ModelCandidate[]; readonly sawModel: boolean; readonly failedUpstreams: readonly string[] }[] = []; vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts index a89f72a381..c4afaca684 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts @@ -21,8 +21,8 @@ import { move, run } from '@floway-dev/pipeline'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import { directFetcher, type FlagId, type ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ ...(await importOriginal()), diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts index 4ea8356ff6..61c87b89b4 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -24,8 +24,8 @@ import { move, run, type Deferred } from '@floway-dev/pipeline'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ModelEndpoints, SseFrame } from '@floway-dev/protocols/common'; import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesCompactionResult, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; -import type { StubChatProviderCall } from '@floway-dev/test-utils'; +import { directFetcher, type FlagId, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/test-utils'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-drive.ts index e62a17f55e..bba0b7359d 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-drive.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-drive.ts @@ -3,7 +3,8 @@ import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gate import { compose, defineStage, move, run } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf, type ExecuteResult, type OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { providerModelOf, type OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { type ExecuteResult } from '@floway-dev/test-utils'; type Result = ExecuteResult>; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-shim_test.ts index 19efc639f7..d304f95f6a 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-shim_test.ts @@ -6,8 +6,7 @@ import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesOutputFunctionCall, OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesStreamEvent, OpenAIResponsesTool } from '@floway-dev/protocols/openai-responses'; import type { OpenAIResponsesInvocation } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; +import { assertEquals, stubModelCandidate, testTelemetryModelIdentity, eventResult } from '@floway-dev/test-utils'; const MESSAGE_SCHEMA = { type: 'object', diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts index c0c7b85282..c6c3efdf3a 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts @@ -18,7 +18,8 @@ import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../../../ import { compose, defer, defineStage, move, run, type Deferred } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { EventResultMetadata, ExecuteResult, ModelCandidate, OpenAIResponsesInvocation } from '@floway-dev/provider'; +import type { ModelCandidate, OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { type EventResultMetadata, type ExecuteResult } from '@floway-dev/test-utils'; const ANSWER = 'response.chat.openaiResponses'; const STREAMED_USAGE = 'response.chat.openaiResponses.streamedUsage'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts index 0b44561d2d..c21bd557e2 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts @@ -10,8 +10,8 @@ import { createInMemoryImageProcessor, initExternalResourceFetcher, initImagePro import { eventFrame } from '@floway-dev/protocols/common'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesResult, OpenAIResponsesStreamEvent, OpenAIResponsesTool, OpenAIResponsesToolChoice } from '@floway-dev/protocols/openai-responses'; -import { type EventResult, type ExecuteResult, type FlagId, type OpenAIImagesEditsRequest, type OpenAIResponsesInvocation } from '@floway-dev/provider'; -import { assert, assertEquals, assertStringIncludes, stubModelCandidate } from '@floway-dev/test-utils'; +import { type FlagId, type OpenAIImagesEditsRequest, type OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { assert, assertEquals, assertStringIncludes, stubModelCandidate, type EventResult, type ExecuteResult } from '@floway-dev/test-utils'; // Dirty integration harness: mock the model registry so the image backend is a // pair of in-test stubs, then drive the whole dispatcher (function-tool rewrite, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts index ede3a2e247..55556d7117 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts @@ -44,8 +44,8 @@ import type { OpenAIResponsesToolChoice, OpenAIResponsesWebSearchAction, } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type EventResult, type ExecuteResult, type FlagId, type OpenAIResponsesInvocation } from '@floway-dev/provider'; -import { assert, assertEquals, assertFalse, assertRejects, stubModelCandidate } from '@floway-dev/test-utils'; +import { type FlagId, type OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { assert, assertEquals, assertFalse, assertRejects, stubModelCandidate, eventResult, type EventResult, type ExecuteResult } from '@floway-dev/test-utils'; import { translateOpenAIResponsesViaAnthropicMessages, translateOpenAIResponsesViaOpenAIChatCompletions } from '@floway-dev/translate'; const runHostedWebSearch = driveHostedToolStage([webSearchHostedTool]); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts index 3190957413..ed5043286b 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts @@ -12,8 +12,8 @@ import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthrop import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { openaiResponsesResultToEvents, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type FlagId, type ModelCandidate, directFetcher, type ProviderOpenAIResponsesResult, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; +import { type FlagId, type ModelCandidate, directFetcher, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; +import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel, type ProviderOpenAIResponsesResult } from '@floway-dev/test-utils'; // Mock the resolver seam so each test hands the http entry exactly the // provider candidates it wants, optionally with an alias-rules overlay diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts index f6581218fb..dd29378bda 100644 --- a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -31,8 +31,8 @@ import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocol import type { GeminiGenerateContentPayload, GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsResult, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { directFetcher, type FlagId, type ModelCandidate, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import { directFetcher, type FlagId, type ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/test-utils'; vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ ...(await importOriginal()), diff --git a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts index 124e6bc00f..abf1cdd4ab 100644 --- a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts @@ -14,8 +14,8 @@ import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; import { isOwned, move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { directFetcher, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import { directFetcher, type ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderCallResult } from '@floway-dev/test-utils'; vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({ ...(await importOriginal()), diff --git a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts index 8c694df07c..b6905b4bae 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts @@ -14,8 +14,8 @@ import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; import { compose, defineStage, isDeferred, move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { directFetcher, type ModelCandidate, type ProviderCallResult, type UpstreamCallOptions, type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import { directFetcher, type ModelCandidate, type UpstreamCallOptions, type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderCallResult } from '@floway-dev/test-utils'; vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({ ...(await importOriginal()), diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts index 641b2e0dc4..0870b2d4f5 100644 --- a/packages/gateway/__tests__/data-plane/rerank-status_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -14,8 +14,8 @@ import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; import { move, run } from '@floway-dev/pipeline'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; -import { directFetcher, type ModelCandidate, type ProviderRerankCallResult } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import { directFetcher, type ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderRerankCallResult } from '@floway-dev/test-utils'; vi.mock('../../src/data-plane/providers/resolution.ts', () => ({ enumerateModelCandidates: vi.fn(), diff --git a/packages/gateway/__tests__/test-utils/provider-pipeline.ts b/packages/gateway/__tests__/test-utils/provider-pipeline.ts index ea11129195..8eb960821b 100644 --- a/packages/gateway/__tests__/test-utils/provider-pipeline.ts +++ b/packages/gateway/__tests__/test-utils/provider-pipeline.ts @@ -1,7 +1,8 @@ import type { GatewayCtx } from '../../src/data-plane/shared/gateway-ctx.ts'; import { takeHttpResponse } from '@floway-dev/http/pipeline'; import { compose, defineStage, isSecret, move, type Pipeline } from '@floway-dev/pipeline'; -import { directFetcher, identityWrapUpstreamCall, type ProviderCallResult, type ProviderModelFacts, type ProviderOperation, type ProviderOperationPayloads, type ProviderResponse, type ProviderRerankResponse, type ProviderRequest, type UpstreamCallOptions } from '@floway-dev/provider'; +import { directFetcher, identityWrapUpstreamCall, type ProviderModelFacts, type ProviderOperation, type ProviderOperationPayloads, type ProviderResponse, type ProviderRerankResponse, type ProviderRequest, type UpstreamCallOptions } from '@floway-dev/provider'; +import { type ProviderCallResult } from '@floway-dev/test-utils'; export const stubProviderPipeline = ( operation: O, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/response-resource.ts b/packages/gateway/src/data-plane/chat/openai-responses/response-resource.ts index d940b93670..c84878fbbe 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/response-resource.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/response-resource.ts @@ -33,12 +33,10 @@ import type { // rows carried by `response.output_item.done` and a snapshot record of their // ordered ids, and awaits that before the terminal frame leaves it, so no // response resource is ever stored. Affinity egress runs underneath too, and -// billing reads `billableUsage` off the `ExecuteResult` rather than the -// resource. One reader sits above this stage, the non-streaming `settle` -// call, and it reads only `status`, which rides through from the upstream and -// is never stated here. The WebSocket transport buffers the terminal event -// and flushes it last, branching only on the event type; it reads nothing off -// the resource at all. +// billing uses its own measured usage facts. Client collection runs above +// this projection, so a mandatory wire default cannot become an upstream +// usage observation. The WebSocket transport buffers the terminal event +// and flushes it last, branching on its type. // // #125's "no tools synthesized when upstream omits it" therefore still holds: // the interior resource carries no `tools`, and egress states `[]` on the way diff --git a/packages/gateway/src/data-plane/tools/web-search/providers/shared.ts b/packages/gateway/src/data-plane/tools/web-search/providers/shared.ts index 1af1100865..110f29b166 100644 --- a/packages/gateway/src/data-plane/tools/web-search/providers/shared.ts +++ b/packages/gateway/src/data-plane/tools/web-search/providers/shared.ts @@ -1,7 +1,7 @@ import { isJsonObject } from '../../../../shared/json-helpers.ts'; import { sleep } from '../../../../shared/sleep.ts'; import type { WebSearchProviderErrorCode, WebSearchProviderResult } from '../types.ts'; -import { discardUpstreamResponse } from '@floway-dev/provider'; +import { discardUpstreamResponse } from '@floway-dev/http'; const MAX_WEB_SEARCH_QUERY_LENGTH = 1000; const RETRY_DELAYS_MS = [1000, 2000, 4000, 8000] as const; diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index 99f0db586c..d529ad0a37 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -21,7 +21,7 @@ export type DumpUpstreamRef = z.infer; // What went wrong on a failed turn. Either a categorized api-error envelope // (real upstream non-2xx or a gateway-synthesized envelope — `kind` matches -// `ApiErrorResult.source`) or an uncategorized failure (anything the +// the upstream or gateway origin) or an uncategorized failure (anything the // transport edge caught or observed mid-flight: thrown // exceptions, source-emitted error events, downstream cancels, write // errors) carrying its one-line reason text. The categorized form stores diff --git a/packages/http/src/index.ts b/packages/http/src/index.ts index 3935f33271..47eb4ea6cc 100644 --- a/packages/http/src/index.ts +++ b/packages/http/src/index.ts @@ -39,3 +39,5 @@ export { HttpProtocolError } from './errors.ts'; export type { HttpProtocolErrorCode } from './errors.ts'; export { STATUS_LINE } from './grammar.ts'; + +export { discardUpstreamResponse } from './response.ts'; diff --git a/packages/http/src/response.ts b/packages/http/src/response.ts index 8b0a1a3dbd..f5d73b1f26 100644 --- a/packages/http/src/response.ts +++ b/packages/http/src/response.ts @@ -19,3 +19,7 @@ export const exchangeResponse = (exchange: HttpResponseExchange): Response => statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]), }); + +export const discardUpstreamResponse = async (response: Response): Promise => { + await response.body?.cancel(); +}; diff --git a/packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts b/packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts index 370d934e1c..4495129725 100644 --- a/packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts +++ b/packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts @@ -2,8 +2,8 @@ import { test } from 'vitest'; import { backfillRequiredFields } from '../../src/stages/backfill-required-fields.ts'; import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderModel, ProviderStreamResult } from '@floway-dev/provider'; -import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import type { ProviderModel } from '@floway-dev/provider'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; diff --git a/packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts b/packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts index 216ca69e52..c10438980d 100644 --- a/packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts +++ b/packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts @@ -2,8 +2,7 @@ import { test } from 'vitest'; import { hoistUserSystemToMessages } from '../../src/stages/hoist-user-system-to-messages.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; diff --git a/packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts b/packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts index 9c3150ce75..643d4d8413 100644 --- a/packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts @@ -3,8 +3,7 @@ import { test } from 'vitest'; import { CLAUDE_CLI_VERSION } from '../../src/headers.ts'; import { injectBillingBlock } from '../../src/stages/inject-billing-block.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; diff --git a/packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts b/packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts index 6e748674d4..d880a86446 100644 --- a/packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts @@ -3,8 +3,7 @@ import { test } from 'vitest'; import { injectDefaultTemplate } from '../../src/stages/inject-default-template.ts'; import { DEFAULT_TEMPLATE_BLOCK, IDENTITY_BLOCK } from '../../src/system-blocks.ts'; import type { AnthropicMessagesClientTool, AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; diff --git a/packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts b/packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts index 38ffb1dff4..d3ea13a2ff 100644 --- a/packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts @@ -3,8 +3,7 @@ import { test } from 'vitest'; import { injectIdentityBlock } from '../../src/stages/inject-identity-block.ts'; import { IDENTITY_BLOCK } from '../../src/system-blocks.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; diff --git a/packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts b/packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts index 964cb648b8..306e32d231 100644 --- a/packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts +++ b/packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts @@ -5,8 +5,7 @@ import { parseMetadataUserID } from '../../src/detection.ts'; import { hoistUserSystemToMessages } from '../../src/stages/hoist-user-system-to-messages.ts'; import { synthesizeMetadataUserId } from '../../src/stages/synthesize-metadata-user-id.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import type { AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; diff --git a/packages/provider-claude-code/__tests__/test-utils/call.ts b/packages/provider-claude-code/__tests__/test-utils/call.ts index 7b19e17638..fcf95c6e34 100644 --- a/packages/provider-claude-code/__tests__/test-utils/call.ts +++ b/packages/provider-claude-code/__tests__/test-utils/call.ts @@ -6,8 +6,8 @@ import { retryClaudeCodeAccess } from '../../src/stages/retry-access.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose, defineStage, move, type RunServices } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { observeProviderCall, type Provider, type ProviderModel, type ProviderStreamResult, type AnthropicMessagesUpstreamCallOptions } from '@floway-dev/provider'; -import { collectChatProviderPipeline } from '@floway-dev/test-utils'; +import { observeProviderCall, type Provider, type ProviderModel, type AnthropicMessagesUpstreamCallOptions } from '@floway-dev/provider'; +import { collectChatProviderPipeline, type ProviderStreamResult } from '@floway-dev/test-utils'; interface Options { upstreamId: string; model: ProviderModel; body: Omit; shaped: boolean; signal?: AbortSignal; call: AnthropicMessagesUpstreamCallOptions; observers?: Pick } export const callClaudeCodeAnthropicMessages = async (opts: Options): Promise> => { diff --git a/packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts b/packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts index 70527099c4..f78a945651 100644 --- a/packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts +++ b/packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts @@ -2,8 +2,7 @@ import { test } from 'vitest'; import { injectCodexDefaultInstructions } from '../../src/stages/inject-default-instructions.ts'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIResponsesProbe, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, assertEquals, stubProviderModel, type ProviderStreamResult } from '@floway-dev/test-utils'; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test', headers: new Headers() }); diff --git a/packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts b/packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts index 88b4923f30..3e8b305b8c 100644 --- a/packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts +++ b/packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts @@ -2,8 +2,7 @@ import { test } from 'vitest'; import { stripCodexUnsupportedFields } from '../../src/stages/strip-unsupported-response-fields.ts'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIResponsesProbe, assertEquals, assertFalse, stubProviderModel } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, assertEquals, assertFalse, stubProviderModel, type ProviderStreamResult } from '@floway-dev/test-utils'; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test', headers: new Headers() }); diff --git a/packages/provider-codex/__tests__/test-utils/call.ts b/packages/provider-codex/__tests__/test-utils/call.ts index ddf08439c0..5563c026ef 100644 --- a/packages/provider-codex/__tests__/test-utils/call.ts +++ b/packages/provider-codex/__tests__/test-utils/call.ts @@ -3,8 +3,8 @@ import { createCodexPipelines } from '../../src/pipelines.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { Provider, ProviderModel, ProviderStreamResult, ProviderOperationPayloads, UpstreamCallOptions } from '@floway-dev/provider'; -import { callProviderPipeline, collectChatProviderPipeline } from '@floway-dev/test-utils'; +import type { Provider, ProviderModel, ProviderOperationPayloads, UpstreamCallOptions } from '@floway-dev/provider'; +import { callProviderPipeline, collectChatProviderPipeline, type ProviderStreamResult } from '@floway-dev/test-utils'; interface CallOptions extends CodexBackendCallBase { model: ProviderModel } interface ResponsesOptions extends CallOptions { body: Omit } diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts index f2b363fa09..dcd4e92386 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotAnthropicMessagesApplyTopLevelCacheControl } from '../../../src/stages/anthropic-messages/apply-top-level-cache-control.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts index 9a9749ee7a..9f53ffc937 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts @@ -4,9 +4,7 @@ import { copilotAnthropicMessagesCompressImages } from '../../../src/stages/anth import { type ImageDimensions, type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { AnthropicMessagesImageBlock, AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesToolResultBlock } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts index f0d03659e8..f764bcd3f7 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotAnthropicMessagesSpeedFast } from '../../../src/stages/anthropic-messages/handle-speed-fast.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesUsage } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const collect = async (events: AsyncIterable): Promise => { diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts index 1549c1be03..ea1ed2a772 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts @@ -3,8 +3,7 @@ import { test } from 'vitest'; import { copilotAnthropicMessagesNormalizeAnthropicBeta } from '../../../src/stages/anthropic-messages/normalize-anthropic-beta.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventResult, type ExecuteResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, eventResult, type ExecuteResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts index b58438264c..852978f32d 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { resolveAnthropicMessagesDownstreamThinkingDisplay, copilotAnthropicMessagesThinkingDisplay } from '../../../src/stages/anthropic-messages/promote-thinking-display.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const collect = async (events: AsyncIterable): Promise => { diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts index 820024ead1..2adfb96e7f 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts @@ -4,9 +4,7 @@ import { CLAUDE_AGENT_USER_AGENT } from '../../../src/auth.ts'; import { copilotAnthropicMessagesSetClaudeAgentHeaders } from '../../../src/stages/anthropic-messages/set-claude-agent-headers.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts index fb85af9cf5..78c22dacbc 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotAnthropicMessagesSetCompactHeaders } from '../../../src/stages/anthropic-messages/set-compact-headers.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts index f1fb671c0a..e1e9903b97 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotAnthropicMessagesSetInitiatorHeader } from '../../../src/stages/anthropic-messages/set-initiator-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts index d6e82b7613..8fd6793abd 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts @@ -4,9 +4,7 @@ import { test } from 'vitest'; import { copilotAnthropicMessagesSetInteractionIdHeader } from '../../../src/stages/anthropic-messages/set-interaction-id-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts index c186aba4dd..50f4bd1fcd 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotAnthropicMessagesSetVisionHeader } from '../../../src/stages/anthropic-messages/set-vision-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts index f01b5d1c0f..c2a72743dc 100644 --- a/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotAnthropicMessagesStripCacheControlExtensions } from '../../../src/stages/anthropic-messages/strip-cache-control-extensions.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type AnthropicMessagesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts index f7b3a3c7e5..3ae90bf500 100644 --- a/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts @@ -4,9 +4,7 @@ import { copilotOpenAIChatCompletionsAbortToolWhitespace } from '../../../src/st import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/tool-argument-whitespace.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, assertStringIncludes, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const invocation = (): OpenAIChatCompletionsProbe => ({ diff --git a/packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts index edd563e987..a03ab56431 100644 --- a/packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { type CopilotCacheableMessage, copilotOpenAIChatCompletionsAttachCacheControlMarkers } from '../../../src/stages/openai-chat-completions/attach-cache-control-markers.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, assertFalse, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts index 62ebbff8cc..9bf8687b2b 100644 --- a/packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts @@ -4,9 +4,7 @@ import { copilotOpenAIChatCompletionsCompressImages } from '../../../src/stages/ import { type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts index 7b83b2e4af..923844ab6a 100644 --- a/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotOpenAIChatCompletionsSetInitiatorHeader } from '../../../src/stages/openai-chat-completions/set-initiator-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts index 5becb700c7..5f540136d9 100644 --- a/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotOpenAIChatCompletionsSetVisionHeader } from '../../../src/stages/openai-chat-completions/set-vision-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIChatCompletionsProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts index fcc28be1da..405fc04faa 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts @@ -4,8 +4,7 @@ import { copilotOpenAIResponsesAbortToolWhitespace } from '../../../src/stages/o import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/tool-argument-whitespace.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, type ProviderOpenAIResponsesResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; const invocation = (): OpenAIResponsesProbe => ({ diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts index 4aa09fa243..061d7beddb 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts @@ -4,9 +4,7 @@ import { copilotOpenAIResponsesCompressImages } from '../../../src/stages/openai import { type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputImage, OpenAIResponsesInputItem, OpenAIResponsesStreamEvent, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts index 438636a614..3009aad07b 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotOpenAIResponsesForceStoreFalse } from '../../../src/stages/openai-responses/force-store-false.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts index 6ba3187951..7b04832154 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts @@ -4,8 +4,7 @@ import { unwrapCopilotItemId, wrapCopilotItemId } from '../../../src/opaque-item import { copilotOpenAIResponsesItemIdMembrane } from '../../../src/stages/openai-responses/item-id-membrane.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { openaiResponsesResultToEvents, type OpenAIResponsesInputItem, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, type ProviderOpenAIResponsesResult } from '@floway-dev/test-utils'; import { stubProviderModel } from '@floway-dev/test-utils'; const invocation = (input: OpenAIResponsesInputItem[] = []): OpenAIResponsesProbe => ({ diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts index 50671ad72a..1f1ead9086 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotOpenAIResponsesSetInitiatorHeader } from '../../../src/stages/openai-responses/set-initiator-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts index 99b66aa8aa..28119e6db5 100644 --- a/packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts @@ -3,9 +3,7 @@ import { test } from 'vitest'; import { copilotOpenAIResponsesSetVisionHeader } from '../../../src/stages/openai-responses/set-vision-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputItem, OpenAIResponsesStreamEvent, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; +import { applyProviderStage, type OpenAIResponsesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const okEvents = (): Promise>> => diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts index 206aecb4ab..61f26175de 100644 --- a/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts @@ -15,11 +15,6 @@ import { replaceHttpHeader } from '@floway-dev/provider'; * The header name is lowercase `x-initiator`; HTTP header names are * case-insensitive on the wire, so the casing is cosmetic. * - * Generic in the run-result type so the count_tokens boundary chain - * (`Response`) and the streaming Anthropic Messages boundary chain (`ExecuteResult<...>`) - * can share one definition, matching the established behavior where - * x-initiator is set on every Copilot Anthropic Messages HTTP call. - * * References: * - https://github.com/caozhiyuan/copilot-api/blob/master/src/services/copilot/create-chat-completions.ts */ diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts index c18924ba60..a4c1b8b929 100644 --- a/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts @@ -6,16 +6,10 @@ import { replaceHttpHeader } from '@floway-dev/provider'; /** * Copilot rejects Anthropic `image` blocks as plain text unless the private * `copilot-vision-request: true` header is set. Detection must scan the final - * post-mutation payload (after other Anthropic Messages boundary stages have run) + * shaped request payload (after other Anthropic Messages boundary stages have run) * and cover both the top-level `message.content` and the nested * `tool_result.content[]` shape; Anthropic allows images in both positions. * - * Generic in the run-result type because the Copilot provider historically - * applied equivalent vision detection to every Anthropic Messages HTTP exchange (chat - * AND count_tokens). Keeping a single generic stage lets both the streaming - * Anthropic Messages boundary chain (`ExecuteResult<...>`) and the count_tokens chain - * (`Response`) share one definition. - * * References: * - https://github.com/caozhiyuan/copilot-api/commit/1f6b98924ae092db9b2010846c32e5cbf10817df */ diff --git a/packages/provider-ollama/src/fetch-models.ts b/packages/provider-ollama/src/fetch-models.ts index 5a9c0e7c3c..63b2223c17 100644 --- a/packages/provider-ollama/src/fetch-models.ts +++ b/packages/provider-ollama/src/fetch-models.ts @@ -20,7 +20,8 @@ import type { OllamaUpstreamConfig } from './config.ts'; import { ollamaFetchShow, ollamaFetchTags } from './control-http.ts'; -import { discardUpstreamResponse, fetchUpstreamModels, type Fetcher, identityWrapUpstreamCall, jsonRequestBody } from '@floway-dev/provider'; +import { discardUpstreamResponse } from '@floway-dev/http'; +import { fetchUpstreamModels, type Fetcher, identityWrapUpstreamCall, jsonRequestBody } from '@floway-dev/provider'; export interface OllamaRawModel { // The slug Ollama uses everywhere (e.g. `gpt-oss:120b`, `deepseek-v4-flash`, diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index b2f19fd134..e9782ad74d 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -17,24 +17,6 @@ export { mergeHttpHeaders, replaceHttpHeader, withHttpContentType } from './http export type { InternalDebugError } from './error.ts'; export { toInternalDebugError } from './error.ts'; -export type { - ApiErrorResult, - EventResult, - EventResultMetadata, - ExecuteResult, - InternalErrorResult, - PlainResult, -} from './result.ts'; -export { - apiErrorToResponse, - decodeApiErrorBody, - discardUpstreamResponse, - eventResult, - internalErrorResult, - plainResult, - readUpstreamApiError, -} from './result.ts'; - export type { InternalAliasedFrom, InternalModel, @@ -55,12 +37,8 @@ export type { Provider, InboundHeaderMatcher, ProviderInstance, - ProviderCallResult, - ProviderRerankCallResult, ProviderModule, AnthropicMessagesUpstreamCallOptions, - ProviderOpenAIResponsesResult, - ProviderStreamResult, OpenAIResponsesAction, UpstreamCallOptions, } from './provider.ts'; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index 91c661cd1d..62068509a8 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -3,8 +3,6 @@ import type { ModelPrefixConfig } from './model-prefix.ts'; import type { ProviderModel, UpstreamModelsCache, UpstreamProviderKind, UpstreamRecord } from './model.ts'; import type { Fetcher } from './options.ts'; import type { ProviderPipelines } from './pipeline.ts'; -import type { ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; // The dispatched action can change independently of the source endpoint's original intent. export type OpenAIResponsesAction = 'generate' | 'compact'; @@ -31,27 +29,6 @@ export interface Provider { instance: ProviderInstance; } -export interface ProviderCallResult { - response: Response; - modelKey: string; -} - -export interface ProviderRerankCallResult extends ProviderCallResult { - target: RerankTarget; -} - -// Protocol fixtures distinguish decoded frames from an HTTP failure reply. -export type ProviderStreamResult = - | { ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } - | { ok: false; response: Response; modelKey: string }; - -// The observed output shape follows the dispatched action, independently of caller intent. -export type ProviderOpenAIResponsesResult = - | { action: 'generate'; ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } - | { action: 'generate'; ok: false; response: Response; modelKey: string } - | { action: 'compact'; ok: true; result: OpenAIResponsesCompactionResult; modelKey: string } - | { action: 'compact'; ok: false; response: Response; modelKey: string }; - // Per-call options the gateway threads through to the provider. // // `fetcher` is the per-upstream proxy-aware indirection for outbound HTTP. diff --git a/packages/provider/src/result.ts b/packages/provider/src/result.ts deleted file mode 100644 index 79c1ebea95..0000000000 --- a/packages/provider/src/result.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { InternalDebugError } from './error.ts'; -import type { PerformanceTelemetryContext, TelemetryModelIdentity } from './telemetry.ts'; -import type { BillableUsage } from '@floway-dev/protocols/common'; - -export interface EventResult { - type: 'events'; - events: AsyncIterable; - modelIdentity: TelemetryModelIdentity; - performance?: PerformanceTelemetryContext; - finalMetadata?: Promise; - // Raw upstream response headers for the source-side `respond` layer to - // forward (blocklist in gateway `shared/respond.ts`). Absent on - // lifted/synthesized streams that have no upstream Response behind them. - headers?: Headers; -} - -export interface EventResultMetadata { - modelIdentity: TelemetryModelIdentity; - performance?: PerformanceTelemetryContext; - // What the upstream turn cost, read from the upstream's own usage. Absent - // until the stream that reports it has been consumed. - billableUsage?: BillableUsage; -} - -// HTTP-shaped error envelope the respond layer forwards to the client -// verbatim (status + headers + body). `source` distinguishes a real upstream -// non-2xx from a gateway-synthesized envelope (model not routable, missing -// stored item, server-tool input rejected, etc.) so observers like the -// request dump can record the failure category truthfully rather than -// labelling every 4xx as `upstream error N`. `upstreamId` is set on real -// upstream 4xx/5xx (`source === 'upstream'`) so the dump row can attribute the -// failure to its source; it is absent on gateway-synthesized envelopes that -// never reached an upstream. -export interface ApiErrorResult { - type: 'api-error'; - source: 'upstream' | 'gateway'; - status: number; - headers: Headers; - body: Uint8Array; - performance?: PerformanceTelemetryContext; - upstreamId?: string; -} - -// Gateway-side bug surface (parser crash, stage throw, etc.). The -// protocol's respond layer renders a debug envelope around `error` -// (stack, cause, target_api) rather than passing through a wire body — -// the shape differs from `ApiErrorResult` for that reason. -export interface InternalErrorResult { - type: 'internal-error'; - status: number; - error: InternalDebugError; - performance?: PerformanceTelemetryContext; -} - -// A fully-shaped non-streaming success body — the output of a source endpoint -// that measures rather than generates (count_tokens). It is NOT an -// `ExecuteResult`: the target emit/stage layer never produces one. The -// orchestrator passes it straight to `respond` without persistence, and -// `respond` emits it verbatim. `upstreamId` is present when the body came from -// a real upstream call and absent for gateway-synthesized envelopes (rewrite -// failures, internal-debug bodies). -export interface PlainResult { - type: 'plain'; - status: number; - headers: Headers; - body: Uint8Array; - upstreamId?: string; -} - -export type ExecuteResult = EventResult | ApiErrorResult | InternalErrorResult; - -export interface EventResultOptions { - performance?: PerformanceTelemetryContext; - finalMetadata?: Promise; - headers?: Headers; -} - -export const eventResult = ( - events: AsyncIterable, - modelIdentity: TelemetryModelIdentity, - options: EventResultOptions = {}, -): EventResult => { - const result: EventResult = { type: 'events', events, modelIdentity }; - if (options.performance !== undefined) result.performance = options.performance; - if (options.finalMetadata !== undefined) result.finalMetadata = options.finalMetadata; - if (options.headers !== undefined) result.headers = options.headers; - return result; -}; - -export const internalErrorResult = (status: number, error: InternalDebugError, performance?: PerformanceTelemetryContext): InternalErrorResult => ({ - type: 'internal-error', - status, - error, - ...(performance ? { performance } : {}), -}); - -export const plainResult = (status: number, headers: Headers, body: Uint8Array, upstreamId?: string): PlainResult => ({ - type: 'plain', - status, - headers, - body, - ...(upstreamId !== undefined ? { upstreamId } : {}), -}); - -export const readUpstreamApiError = async (response: Response, upstreamId?: string): Promise => ({ - type: 'api-error', - source: 'upstream', - status: response.status, - headers: new Headers(response.headers), - body: new Uint8Array(await response.arrayBuffer()), - ...(upstreamId !== undefined ? { upstreamId } : {}), -}); - -// A response whose body is neither read nor cancelled keeps its transport -// alive: on the direct-connect egress the dialed socket is closed only from the -// body's own read-to-end or cancel path, so an abandoned response strands one -// socket for the life of the process. Every path that walks away from an -// upstream response goes through here. -export const discardUpstreamResponse = async (response: Response): Promise => { - await response.body?.cancel(); -}; - -export const apiErrorToResponse = (error: ApiErrorResult): Response => - new Response(error.body.slice().buffer, { - status: error.status, - headers: new Headers(error.headers), - }); - -export const decodeApiErrorBody = (error: ApiErrorResult): string => new TextDecoder().decode(error.body); diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index f2c0b82448..1a0364c764 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -12,6 +12,7 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", + "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*" } } diff --git a/packages/test-utils/src/chat-provider-pipeline.ts b/packages/test-utils/src/chat-provider-pipeline.ts index 845b7bf864..8b43512d05 100644 --- a/packages/test-utils/src/chat-provider-pipeline.ts +++ b/packages/test-utils/src/chat-provider-pipeline.ts @@ -36,8 +36,9 @@ export const collectChatProviderPipeline = async ); if (exchange.body !== null) setRelease(exchange.body, async () => {}); } else if (output === null) { - const bytes = await exchangeResponse(exchange).arrayBuffer(); + const raw = exchangeResponse(exchange); if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const bytes = await raw.arrayBuffer(); response = new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]) }); } } catch (error) { diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index e129c80457..76abeb9966 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -1,3 +1,4 @@ +export { eventResult, type EventResult, type EventResultMetadata, type ExecuteResult, type ProviderCallResult, type ProviderRerankCallResult, type ProviderStreamResult, type ProviderOpenAIResponsesResult } from './provider-results.ts'; export { collectPreparedProviderRequest } from './prepared-provider-request.ts'; export { collectHttpPipeline } from './http-pipeline.ts'; export { applyProviderStage, type ProviderStageProbe, type OpenAIChatCompletionsProbe, type OpenAIResponsesProbe, type AnthropicMessagesProbe } from './provider-stage.ts'; diff --git a/packages/test-utils/src/provider-pipeline.ts b/packages/test-utils/src/provider-pipeline.ts index 2cb2ea9102..9b1b75f77b 100644 --- a/packages/test-utils/src/provider-pipeline.ts +++ b/packages/test-utils/src/provider-pipeline.ts @@ -1,7 +1,8 @@ +import type { ProviderCallResult, ProviderRerankCallResult } from './provider-results.ts'; import { noopUpstreamCallOptions } from './stubs.ts'; import { exchangeResponse, type HttpServices } from '@floway-dev/http/pipeline'; import { move, run, setRelease } from '@floway-dev/pipeline'; -import type { NonChatProviderOperation, Provider, ProviderCallResult, ProviderModel, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderRerankCallResult, ProviderOperationRequest, UpstreamCallOptions } from '@floway-dev/provider'; +import type { NonChatProviderOperation, Provider, ProviderModel, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderOperationRequest, UpstreamCallOptions } from '@floway-dev/provider'; import { providerModelFacts } from '@floway-dev/provider'; type CallResult = @@ -32,13 +33,17 @@ export const callProviderPipeline = async ( throw exchange.error; } const response = exchangeResponse(exchange); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); let bytes: ArrayBuffer; try { bytes = await response.arrayBuffer(); - if (exchange.body !== null) setRelease(exchange.body, async () => {}); - } finally { - await executed.drain(); + } catch (error) { + try { await executed.drain(); } catch (cleanupError) { + if (cleanupError !== error) throw new AggregateError([error, cleanupError], 'Provider test call and cleanup failed', { cause: error }); + } + throw error; } + await executed.drain(); const result = { response: new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: response.headers }), modelKey: facts['response.provider.modelKey'], diff --git a/packages/test-utils/src/provider-results.ts b/packages/test-utils/src/provider-results.ts new file mode 100644 index 0000000000..5847caf322 --- /dev/null +++ b/packages/test-utils/src/provider-results.ts @@ -0,0 +1,41 @@ +import type { BillableUsage, ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; +import type { OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { InternalDebugError, PerformanceTelemetryContext, TelemetryModelIdentity } from '@floway-dev/provider'; + +export interface ProviderCallResult { + response: Response; + modelKey: string; +} +export interface ProviderRerankCallResult extends ProviderCallResult { target: RerankTarget } +export type ProviderStreamResult = + | { ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } + | { ok: false; response: Response; modelKey: string }; +export type ProviderOpenAIResponsesResult = + | (ProviderStreamResult & { action: 'generate' }) + | { action: 'compact'; ok: true; result: OpenAIResponsesCompactionResult; modelKey: string } + | { action: 'compact'; ok: false; response: Response; modelKey: string }; + +export interface EventResultMetadata { + modelIdentity: TelemetryModelIdentity; + performance?: PerformanceTelemetryContext; + billableUsage?: BillableUsage; +} +export interface EventResult { + type: 'events'; + events: AsyncIterable; + modelIdentity: TelemetryModelIdentity; + performance?: PerformanceTelemetryContext; + finalMetadata?: Promise; + headers?: Headers; +} +export type ExecuteResult = EventResult + | { type: 'api-error'; source: 'upstream' | 'gateway'; status: number; headers: Headers; body: Uint8Array; performance?: PerformanceTelemetryContext; upstreamId?: string } + | { type: 'internal-error'; status: number; error: InternalDebugError; performance?: PerformanceTelemetryContext }; + +export const eventResult = (events: AsyncIterable, modelIdentity: TelemetryModelIdentity, options: Omit, 'type' | 'events' | 'modelIdentity'> = {}): EventResult => { + const result: EventResult = { type: 'events', events, modelIdentity }; + if (options.performance !== undefined) result.performance = options.performance; + if (options.finalMetadata !== undefined) result.finalMetadata = options.finalMetadata; + if (options.headers !== undefined) result.headers = options.headers; + return result; +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1b3f4ffd4f..19fe7cf303 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -661,6 +661,9 @@ importers: '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline + '@floway-dev/protocols': + specifier: workspace:* + version: link:../protocols '@floway-dev/provider': specifier: workspace:* version: link:../provider From 776b4627985078bae8f8be5eb6e948e64ca378a3 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:04:09 +0800 Subject: [PATCH 147/156] fix: select recorded classic Completions client streams Resolve the authoritative rendered stream for classic Completions alongside the chat client-frame fact. Inspect the stored stream tag so ordinary JSON extensions named stream cannot select a recording. Verified 18 focused request UI tests, web typecheck and scoped lint, including classic collected output, stream.end proof and non-streaming JSON non-selection. --- .../components/requests/run-stream_test.ts | 21 +++++++++++++++++++ .../components/requests/stream-render_test.ts | 10 +++++++++ .../web/src/components/requests/run-stream.ts | 9 +++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/apps/web/__tests__/components/requests/run-stream_test.ts b/apps/web/__tests__/components/requests/run-stream_test.ts index 0f8a677945..29bf128ebd 100644 --- a/apps/web/__tests__/components/requests/run-stream_test.ts +++ b/apps/web/__tests__/components/requests/run-stream_test.ts @@ -40,3 +40,24 @@ test('keeps a selected empty stream and distinguishes it from no client stream', { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': null } }, ])))).toBeNull(); }); + +test('selects the rendered classic Completions stream and its own end proof', () => { + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emitOpenAICompletions', parentStageId: null, facts: {} }, + { type: 'stream.frame', streamId: 1, frames: [{ type: 'event', event: { choices: [{ text: 'upstream' }] } }] }, + { type: 'stream.frame', streamId: 2, frames: [{ type: 'event', event: { id: 'completion-1', object: 'text_completion', created: 1, model: 'm', choices: [{ index: 0, text: 'client', finish_reason: 'stop', logprobs: null }] } }, { type: 'done' }] }, + { type: 'stream.end', streamId: 2 }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.openaiCompletions.rendered': streamFact(2) } }, + ])); + expect(clientStreamOf(events)).toMatchObject({ events: [{ frame: { event: { choices: [{ text: 'client' }] } } }, { frame: { type: 'done' } }], ended: true }); +}); + +test('does not select a JSON Completions extension that resembles a decoded stream reference', () => { + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emitOpenAICompletions', parentStageId: null, facts: {} }, + { type: 'stream.frame', streamId: 7, frames: [{ type: 'event', event: { text: 'unselected' } }] }, + { type: 'stream.end', streamId: 7 }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.openaiCompletions.rendered': { stream: 7, $stream: 7 } } }, + ])); + expect(clientStreamOf(events)).toBeNull(); +}); diff --git a/apps/web/__tests__/components/requests/stream-render_test.ts b/apps/web/__tests__/components/requests/stream-render_test.ts index 786f461c36..e959392b0d 100644 --- a/apps/web/__tests__/components/requests/stream-render_test.ts +++ b/apps/web/__tests__/components/requests/stream-render_test.ts @@ -58,3 +58,13 @@ test('renders and copies final Anthropic citation wire fields once while collect expect(copied).not.toContain('"url"'); expect(await collectStream('anthropic-messages', stream)).toMatchObject({ result: { content: [{ type: 'text', text: 'answer', citations: [citation, citation] }] }, error: null, truncated: false }); }); + +test('collects the classic Completions stream published by its rendered fact', async () => { + const stream = clientStreamOf(toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emitOpenAICompletions', parentStageId: null, facts: {} }, + { type: 'stream.frame', streamId: 4, frames: [eventFrame({ id: 'completion-1', object: 'text_completion', created: 1, model: 'm', choices: [{ index: 0, text: 'classic answer', finish_reason: 'stop', logprobs: null }] }), { type: 'done' }] }, + { type: 'stream.end', streamId: 4 }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.openaiCompletions.rendered': streamFact(4) } }, + ])))!; + expect(await collectStream('openai-completions', stream)).toMatchObject({ result: { choices: [{ text: 'classic answer' }] }, error: null, truncated: false }); +}); diff --git a/apps/web/src/components/requests/run-stream.ts b/apps/web/src/components/requests/run-stream.ts index d22b7a1019..512555de8d 100644 --- a/apps/web/src/components/requests/run-stream.ts +++ b/apps/web/src/components/requests/run-stream.ts @@ -16,9 +16,12 @@ export const clientStreamOf = (ndjson: string): RecordedClientStream | null => { if (line.length === 0) continue; const event = JSON.parse(line) as DumpEvent; const decoded = read(event); - if (decoded?.facts && 'response.chat.clientFrames' in decoded.facts) { - const selected = decoded.facts['response.chat.clientFrames'] as { stream: number } | null; - client = selected?.stream ?? null; + if ((event.type === 'stage.entered' || event.type === 'stage.leaved') && event.facts !== undefined) { + for (const key of ['response.chat.clientFrames', 'response.openaiCompletions.rendered']) { + if (!(key in event.facts)) continue; + const selected = event.facts[key]; + client = selected !== null && typeof selected === 'object' && '$stream' in selected ? selected.$stream as number : null; + } } if (event.type === 'stream.frame' || event.type === 'stream.end') { const stream = streams.get(event.streamId) ?? { events: [], ended: false }; From 43e277f8c6585028792d622ad84139f4675b5e54 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:05:51 +0800 Subject: [PATCH 148/156] Record native web search backend failures in the owning stage Await the scoped logger with the original error before projecting the native unavailable tool result. Preserve client semantics and verify the stage log retains the exact error and cause without introducing global error storage. --- .../web-search-shim_test.ts | 47 ++++++++++++++++++- .../anthropic-messages/web-search-shim.ts | 10 ++-- .../anthropic-messages/web-search-tool.ts | 1 + 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts index 45562f2330..02850c9929 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts @@ -1,6 +1,7 @@ -import { test } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { driveWebSearchStage } from './web-search-drive.ts'; +import * as webSearchShim from '../../../../src/data-plane/chat/anthropic-messages/web-search-shim.ts'; import { decodeWebSearchCitationPayload, decodeWebSearchResultPayload, @@ -10,11 +11,13 @@ import { prepareAnthropicMessagesWebSearchShimRequest, rewriteAnthropicMessagesWebSearchEventsToNative, } from '../../../../src/data-plane/chat/anthropic-messages/web-search-shim.ts'; +import { runAnthropicMessagesWebSearchTool } from '../../../../src/data-plane/chat/anthropic-messages/web-search-tool.ts'; import { DEFAULT_WEB_SEARCH_CONFIG } from '../../../../src/data-plane/tools/web-search/config.ts'; import type { WebSearchProvider, WebSearchProviderResult } from '../../../../src/data-plane/tools/web-search/types.ts'; import { initRepo } from '../../../../src/repo/index.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move, run, type Event as RunEvent } from '@floway-dev/pipeline'; import { anthropicMessagesProtocolFrameToSSEFrame } from '@floway-dev/protocols/anthropic-messages'; import type { AnthropicMessagesAssistantContentBlock, @@ -826,12 +829,14 @@ const runStreamingShim = ( events: AnthropicMessagesStreamEvent[], state: AnthropicMessagesWebSearchShimState, provider?: ReturnType, + reportFailure: (error: unknown) => Promise = async () => {}, ) => collectStreamEvents( rewriteAnthropicMessagesWebSearchEventsToNative( toAsyncIterable(events.map(event => ({ type: 'event' as const, event }))), state, provider, + reportFailure, ), ); @@ -1328,3 +1333,43 @@ test('rewriteAnthropicMessagesWebSearchEventsToNative requires a provider when m 'Active messages web-search rewrite requires a provider.', ); }); + +test('the native web-search tool records its original backend error before serving unavailable', async () => { + initRepo(new InMemoryRepo()); + const fault = new Error('search backend unavailable', { cause: new TypeError('backend connection state') }); + const provider = activeProvider(searchOnlyProvider(async () => { throw fault; })); + const selected = stubModelCandidate({ enabledFlags: new Set(['anthropic-messages-web-search-shim']) }); + const backend = vi.spyOn(webSearchShim, 'resolveActiveAnthropicMessagesWebSearchProvider').mockResolvedValue(provider); + try { + const terminal = defineStage, Record>({ + name: 'nativeSearchSource', + return: { provides: ['response.chat.anthropicMessages', 'response.chat.anthropicMessages.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + execute: async facts => move({ + ...facts, 'response.chat.anthropicMessages': { + kind: 'stream', frames: toAsyncIterable([ + upstreamMessageStart(), ...upstreamWebSearchBlock(0, 'tool_1', 'question'), ...upstreamMessageEnd('tool_use'), + ].map(event => eventFrame(event))), + }, + 'response.chat.anthropicMessages.streamedUsage': null, 'response.usage.billable': [], + 'response.http.headers': [], 'response.http.status': 200, 'response.http.body': null, + }), + }); + const logs: RunEvent[] = []; + const outcome = await run(compose, Record>('nativeSearch', [ + runAnthropicMessagesWebSearchTool({ targetOf: () => 'anthropicMessages' }), terminal, + ]), move({ + 'request.chat.anthropicMessages': { model: 'model', max_tokens: 32, messages: [{ role: 'user', content: 'question' }], tools: [{ type: 'web_search_20260209' }] }, + 'route.attempt': { candidateId: 0, upstreamId: selected.provider.upstreamId, modelId: selected.model.id, flags: ['anthropic-messages-web-search-shim'] }, + }), { gateway: mockChatGatewayCtx(), resolveAttempt: () => selected, dump: async (event: RunEvent) => { logs.push(event); } }); + const answer = outcome.facts['response.chat.anthropicMessages'] as { frames: AsyncIterable> }; + const events = await collectStreamEvents(answer.frames); + expect(events).toContainEqual(expect.objectContaining({ type: 'content_block_start', content_block: expect.objectContaining({ type: 'web_search_tool_result', content: { type: 'web_search_tool_result_error', error_code: 'unavailable' } }) })); + const entry = logs.find(event => event.type === 'stage.log' && event.level === 'error'); + expect(entry).toMatchObject({ context: 'runAnthropicMessagesWebSearchTool', message: 'web search failed' }); + expect(entry?.type === 'stage.log' ? entry.fields?.error : undefined).toBe(fault); + expect(fault.cause).toBeInstanceOf(TypeError); + await outcome.drain(); + } finally { + backend.mockRestore(); + } +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts index a24c947705..8e3bde6fd9 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts @@ -637,6 +637,7 @@ const runWebSearchStopHandler = async function* ( shimState: ShimStreamingState, state: Extract, provider: ActiveAnthropicMessagesWebSearchProvider, + reportFailure: (error: unknown) => Promise, ): AsyncGenerator> { const parsedInput = (() => { if (block.inputJson === '') return null; @@ -684,8 +685,8 @@ const runWebSearchStopHandler = async function* ( }; const providerResult = await runWebSearchAndRecordUsage({ provider: provider.impl, providerName: provider.providerName, keyId: provider.apiKeyId, request }); return buildNativeWebSearchResultBlockFromProviderResult(providerResult, block.upstreamToolUseId); - } catch { - // TODO: Add gateway-side recent web-search error-log storage so operators can inspect detailed provider/runtime failures even though the client-visible native error intentionally collapses them to `unavailable`. + } catch (error) { + await reportFailure(error); return buildNativeWebSearchErrorResultBlock(block.upstreamToolUseId, 'unavailable'); } })(); @@ -707,7 +708,8 @@ const runWebSearchStopHandler = async function* ( export const rewriteAnthropicMessagesWebSearchEventsToNative = async function* ( frames: AsyncIterable>, state: AnthropicMessagesWebSearchShimState, - provider?: ActiveAnthropicMessagesWebSearchProvider, + provider: ActiveAnthropicMessagesWebSearchProvider | undefined, + reportFailure: (error: unknown) => Promise, ): AsyncGenerator> { if (state.mode === 'inactive') { yield* frames; @@ -800,7 +802,7 @@ export const rewriteAnthropicMessagesWebSearchEventsToNative = async function* ( throw new Error('web-search shim entered intercept path without active state.'); } - yield* runWebSearchStopHandler(activeBlock, shimState, state, provider!); + yield* runWebSearchStopHandler(activeBlock, shimState, state, provider!, reportFailure); activeBlock = undefined; continue; } diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts index 180d0a48b1..f348fe75d0 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts @@ -119,6 +119,7 @@ export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => de answer.frames as AsyncIterable>, prepared.state, backend, + async error => { await use.log.error('web search failed', { error }); }, ), }, }), From 8ee0a04210f4655d6ca2169c75c08c14be243bf7 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:36:54 +0800 Subject: [PATCH 149/156] Describe native counting as nonbillable measurement Remove obsolete future-charge assumptions and the claim that counting never dials upstream. Explain why transport proof cannot recover AI usage for native measurement failures. --- .../data-plane/chat/anthropic-messages/http_test.ts | 5 ----- .../src/data-plane/chat/anthropic-messages/count-tokens.ts | 6 ++---- .../data-plane/chat/gemini-generate-content/count-tokens.ts | 6 ++---- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index 97d4a4c7e3..525c681f07 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -235,13 +235,8 @@ test('POST /v1/messages/count_tokens proxies the upstream measurement body', asy assertEquals(body.input_tokens, 99); assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); - // A measurement goes through settlement like every other run. It names no billed entity — - // nothing here is billable today — but the row is written, because "count_tokens is exempt - // from billing" would hard-code today's commercial arrangement into the architecture. An - // upstream that began charging would provide a non-empty set and nothing else would change. await flushBackground(); assertEquals(await repo.usage.listAll(), []); - // No latency to report either: nothing dialled an upstream on a path that measures. assertEquals(await repo.performance.listAll(), []); }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts index 9cea57040f..53f8feeeb3 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -12,10 +12,8 @@ import { compose, type Pipeline } from '@floway-dev/pipeline'; export const anthropicMessagesCountTokensPipeline = (): Pipeline => compose('anthropicMessagesCountTokens', [ - // A measurement goes through settlement like every other run. It provides an empty - // billed set because nothing here is billable today, not because the operation is - // exempt — an upstream that began charging for it would provide a non-empty one and - // nothing else would change. + // Native counts measure a request without generating AI usage; a failed reply must + // not recover a billable entity from the transport-call proof. writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, undefined, false), serializeClientJson('response.chat.anthropicMessages.rendered'), emitAnthropicMessagesTokenCount, diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts index 149534cc29..02e62fc4c3 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -16,10 +16,8 @@ import { compose, type Pipeline } from '@floway-dev/pipeline'; export const geminiGenerateContentCountTokensPipeline = (): Pipeline => compose('geminiGenerateContentCountTokens', [ - // A measurement goes through settlement like every other run. It provides an empty - // billed set because nothing here is billable today, not because the operation is - // exempt — an upstream that began charging for it would provide a non-empty one and - // nothing else would change. + // Native counts measure a request without generating AI usage; a failed reply must + // not recover a billable entity from the transport-call proof. writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, undefined, false), serializeClientJson('response.chat.geminiGenerateContent.rendered'), emitGeminiGenerateContentTokenCount, From 4c578eed6c343b7e58685f060c92d2500c488f61 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:38:53 +0800 Subject: [PATCH 150/156] test: prove historical edge dump retirement preserves run records Seed pre-migration SQLite records and their owned request, response and upstream-response files. Assert the real migration deletes all edge shapes, queues each file for collection, and preserves the complete run row and its owned file. Keep historical schema assertions separate from the retired runtime storage surface. --- .../migrations/drop-edge-dump-records_test.ts | 55 +++++++++++++++++++ .../migrations/dump-upstream-body_test.ts | 12 ++-- 2 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 packages/gateway/__tests__/migrations/drop-edge-dump-records_test.ts diff --git a/packages/gateway/__tests__/migrations/drop-edge-dump-records_test.ts b/packages/gateway/__tests__/migrations/drop-edge-dump-records_test.ts new file mode 100644 index 0000000000..8dbf0016c9 --- /dev/null +++ b/packages/gateway/__tests__/migrations/drop-edge-dump-records_test.ts @@ -0,0 +1,55 @@ +import { DatabaseSync } from 'node:sqlite'; + +import { expect, test } from 'vitest'; + +import { migrationSqlByFilename } from '../repo/test-sqlite.ts'; + +const MIGRATION = '0086_drop_edge_dump_records.sql'; + +test('deletes historical edge records and retires all their files while preserving run records', () => { + const db = new DatabaseSync(':memory:'); + try { + for (const [filename, sql] of migrationSqlByFilename) { + if (filename === MIGRATION) break; + db.exec(sql); + } + db.prepare(`INSERT INTO api_keys (id, user_id, name, key, created_at, server_secret, dump_retention_seconds) + VALUES (?, 1, ?, ?, ?, ?, 3600)`).run('key', 'Migration key', 'migration-key', '2026-01-01T00:00:00Z', '44'.repeat(32)); + const stage = db.prepare(`INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) + VALUES (?, ?, ?, 'staged', 10000)`); + const insert = db.prepare(`INSERT INTO dump_records + (key_id, id, created_at, meta_json, request_headers_json, request_body_descriptor, response_body_descriptor, response_upstream_body_descriptor) + VALUES ('key', ?, 1000, '{}', '[]', ?, ?, ?)`); + const file = (id: string, part: 'request' | 'response' | 'response-upstream', type: string) => { + const key = `dumps/v1/key/${id}-${part}.gz`; + stage.run(key, `dump-${part}`, JSON.stringify(['key', id])); + return JSON.stringify({ key, type }); + }; + insert.run('edge-bytes', file('edge-bytes', 'request', 'bytes'), file('edge-bytes', 'response', 'bytes'), file('edge-bytes', 'response-upstream', 'events')); + insert.run('edge-events', null, file('edge-events', 'response', 'events'), null); + insert.run('edge-empty', file('edge-empty', 'request', 'bytes'), null, null); + insert.run('retained-run', null, file('retained-run', 'response', 'run'), null); + + expect(db.prepare('SELECT id FROM dump_records ORDER BY id').all()).toEqual([ + { id: 'edge-bytes' }, { id: 'edge-empty' }, { id: 'edge-events' }, { id: 'retained-run' }, + ]); + expect(db.prepare('SELECT DISTINCT state, collect_after FROM spilled_files').all()).toEqual([{ state: 'owned', collect_after: null }]); + + const retainedRun = db.prepare("SELECT * FROM dump_records WHERE id = 'retained-run'").get(); + const migration = migrationSqlByFilename.find(([filename]) => filename === MIGRATION)!; + db.exec(migration[1]); + + expect(db.prepare('SELECT id FROM dump_records').all()).toEqual([{ id: 'retained-run' }]); + expect(db.prepare("SELECT * FROM dump_records WHERE id = 'retained-run'").get()).toEqual(retainedRun); + expect(db.prepare('SELECT file_key, state, collect_after FROM spilled_files WHERE owner_key != ? ORDER BY file_key') + .all(JSON.stringify(['key', 'retained-run']))).toEqual([ + { file_key: 'dumps/v1/key/edge-bytes-request.gz', state: 'retired', collect_after: 0 }, + { file_key: 'dumps/v1/key/edge-bytes-response-upstream.gz', state: 'retired', collect_after: 0 }, + { file_key: 'dumps/v1/key/edge-bytes-response.gz', state: 'retired', collect_after: 0 }, + { file_key: 'dumps/v1/key/edge-empty-request.gz', state: 'retired', collect_after: 0 }, + { file_key: 'dumps/v1/key/edge-events-response.gz', state: 'retired', collect_after: 0 }, + ]); + expect(db.prepare('SELECT state, collect_after FROM spilled_files WHERE owner_key = ?') + .get(JSON.stringify(['key', 'retained-run']))).toEqual({ state: 'owned', collect_after: null }); + } finally { db.close(); } +}); diff --git a/packages/gateway/__tests__/migrations/dump-upstream-body_test.ts b/packages/gateway/__tests__/migrations/dump-upstream-body_test.ts index b36265e9fb..b52278a61a 100644 --- a/packages/gateway/__tests__/migrations/dump-upstream-body_test.ts +++ b/packages/gateway/__tests__/migrations/dump-upstream-body_test.ts @@ -6,13 +6,9 @@ import { migrationSqlByFilename } from '../repo/test-sqlite.ts'; const MIGRATION = '0084_dump_upstream_body.sql'; -// The upstream body column is additive: nullable, and the three spilled-files -// triggers from 0066 are rebuilt to cover the new descriptor. The rebuilt -// triggers' behavior is exercised end-to-end by the dump-store round-trip -// tests (an upstream file is staged, adopted to `owned`, and rehydrated); this -// test asserts the migration itself: the column appears, it is nullable, and -// the rebuilt triggers are present by name. -test('0089 adds a nullable response_upstream_body_descriptor column', () => { +// Historical upstream-body storage remains in the schema; the run-only migration +// retires its records and files. The column and registry triggers still exist. +test('0084 adds a nullable response_upstream_body_descriptor column', () => { const before = new DatabaseSync(':memory:'); for (const [filename, sql] of migrationSqlByFilename) { if (filename === MIGRATION) break; @@ -29,7 +25,7 @@ test('0089 adds a nullable response_upstream_body_descriptor column', () => { const col = columnsAfter.find(c => c.name === 'response_upstream_body_descriptor'); expect(col).toBeDefined(); expect(col!.notnull).toBe(0); // nullable - // The rebuilt triggers are present (rebuilt, not dropped) after 0089. + // The rebuilt registry triggers survive the later run-only migration. for (const trigger of [ 'dump_records_validate_spilled_files', 'dump_records_adopt_spilled_files', From ca63c88f30c5ffc60081c2bc34bb9c0c4febb08c Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:42:27 +0800 Subject: [PATCH 151/156] refactor: inspect recorded frames through the stage viewer Retire the unreachable protocol-specific event viewer and its render/copy adapters. The actual stage viewer decodes recorded protocol frames directly; its citation regression verifies both visible source fields and copied frame content. Keep generic event virtualization, search and collapse tests. Prune unused legacy edge columns from cold run detail reads while retaining the schema-required empty headers write. Verified 57 request UI tests, 16 dump repository tests, gateway/web typechecks and scoped lint. --- .../components/requests/events_test.tsx | 21 +++---- .../components/requests/run-stages_test.tsx | 21 +++++++ .../components/requests/stream-render_test.ts | 12 +--- apps/web/src/components/requests/events.tsx | 17 +++--- .../src/components/requests/stream-render.ts | 57 +------------------ packages/gateway/src/repo/dump-store.ts | 7 +-- 6 files changed, 47 insertions(+), 88 deletions(-) diff --git a/apps/web/__tests__/components/requests/events_test.tsx b/apps/web/__tests__/components/requests/events_test.tsx index 1d10630f87..4de2082ea6 100644 --- a/apps/web/__tests__/components/requests/events_test.tsx +++ b/apps/web/__tests__/components/requests/events_test.tsx @@ -1,22 +1,23 @@ import { fireEvent, screen, within } from '@testing-library/react'; import { describe, expect, it } from 'vitest'; -import { EventList } from '../../../src/components/requests/events'; +import { RenderedEventList, type RenderedEvent } from '../../../src/components/requests/events'; import { renderInApp } from '../../render'; -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; -const events: DumpStreamEvent[] = Array.from({ length: 120 }, (_, i) => ({ - ts: i * 15, - frame: { +const events: RenderedEvent[] = Array.from({ length: 120 }, (_, i) => ({ + timestamp: i * 15, + event: null, + parseError: null, + text: JSON.stringify({ type: 'event', event: { id: 'demo', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { content: i === 99 ? 'needle ' : `chunk ${i + 1}` }, finish_reason: null }], }, - }, + }, null, 2), })); -events.push({ ts: 1800, frame: { type: 'done' } }); +events.push({ timestamp: 1800, event: 'done', parseError: null, text: JSON.stringify({ type: 'done' }, null, 2) }); -const renderEvents = () => renderInApp(View selector} />); +const renderEvents = () => renderInApp( event.text).join('\n')} toolbarStart={View selector} />); describe('per-event view', () => { it('renders separate highlighted JSON blocks and collapses each event independently', () => { @@ -41,8 +42,8 @@ describe('per-event view', () => { expect(view.container.querySelector('code')?.textContent).toContain(''); fireEvent.change(screen.getByRole('textbox', { name: 'Search events' }), { target: { value: 'unmatched' } }); expect(screen.getByText('No matching events')).toBeTruthy(); - fireEvent.change(screen.getByRole('textbox', { name: 'Search events' }), { target: { value: '[DONE]' } }); - expect(screen.getByRole('button', { name: '#121 [DONE]' })).toBeTruthy(); + fireEvent.change(screen.getByRole('textbox', { name: 'Search events' }), { target: { value: 'done' } }); + expect(screen.getByRole('button', { name: '#121 done' })).toBeTruthy(); expect(screen.queryByText('Invalid JSON')).toBeNull(); }); }); diff --git a/apps/web/__tests__/components/requests/run-stages_test.tsx b/apps/web/__tests__/components/requests/run-stages_test.tsx index 9812d978f4..6f773d964f 100644 --- a/apps/web/__tests__/components/requests/run-stages_test.tsx +++ b/apps/web/__tests__/components/requests/run-stages_test.tsx @@ -68,4 +68,25 @@ describe('run stages', () => { expect(screen.queryByText(/upstream content/)).toBeNull(); }); + it('shows and copies the recorded Anthropic citation source without wire projection', async () => { + const writeText = vi.fn(async (_text: string) => {}); + Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } }); + const citation = { type: 'search_result_location', source: 'https://citation.test/anthropic', title: 'Reference', search_result_index: 0, start_block_index: 0, end_block_index: 1 }; + const frame = { type: 'event' as const, event: { type: 'content_block_delta', index: 0, delta: { type: 'citations_delta', citation } } }; + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': streamFact(1) } }, + { type: 'stream.frame', streamId: 1, frames: [frame] }, + { type: 'stream.end', streamId: 1 }, + ])); + renderInApp(); + await screen.findByTestId('state'); + fireEvent.click(screen.getByRole('combobox', { name: 'Stage view' })); + fireEvent.click(screen.getByRole('option', { name: 'Protocol frames' })); + expect(screen.getByText(/https:\/\/citation.test\/anthropic/)).toBeTruthy(); + fireEvent.click(screen.getAllByRole('button', { name: 'Copy' })[0]!); + await waitFor(() => expect(writeText).toHaveBeenCalledTimes(1)); + expect(JSON.parse(writeText.mock.calls[0]![0])).toEqual(frame); + }); + }); diff --git a/apps/web/__tests__/components/requests/stream-render_test.ts b/apps/web/__tests__/components/requests/stream-render_test.ts index e959392b0d..f5f92d6439 100644 --- a/apps/web/__tests__/components/requests/stream-render_test.ts +++ b/apps/web/__tests__/components/requests/stream-render_test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import { clientStreamOf } from '../../../src/components/requests/run-stream'; -import { collectStream, renderStreamEvents, streamEventsCopyText } from '../../../src/components/requests/stream-render'; +import { collectStream } from '../../../src/components/requests/stream-render'; import { encodeRun, streamFact, toNdjson, type Event } from '@floway-dev/pipeline'; import { anthropicMessagesEventToSsePayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; @@ -23,7 +23,7 @@ test('collects normal Chat Completions EOF without inventing a done frame', asyn const collected = await collectStream('openai-chat-completions', stream); expect(collected).toMatchObject({ result: { choices: [{ message: { content: 'hello' } }] }, error: null, truncated: false }); expect(stream.events).toHaveLength(1); - expect(streamEventsCopyText('openai-chat-completions', stream.events)).not.toContain('[DONE]'); + expect(stream.events.map(({ frame }) => frame.type)).toEqual(['event']); }); for (const terminal of [false, true]) { @@ -38,7 +38,7 @@ test('separates an ended protocol error from a truncated recording', async () => expect(await collectStream('anthropic-messages', stream)).toMatchObject({ result: null, error: expect.stringContaining('upstream failed'), truncated: false }); }); -test('renders and copies final Anthropic citation wire fields once while collecting the canonical result', async () => { +test('collects final Anthropic citation wire fields into the canonical result', async () => { const citation = { type: 'search_result_location' as const, url: 'https://example.com/source', title: 'Reference', search_result_index: 0, start_block_index: 0, end_block_index: 1, cited_text: 'evidence' }; const events: AnthropicMessagesStreamEvent[] = [ { type: 'message_start', message: { id: 'msg-1', type: 'message', role: 'assistant', content: [], model: 'm', stop_reason: null, stop_sequence: null, usage: { input_tokens: 1, output_tokens: 0 } } }, @@ -50,12 +50,6 @@ test('renders and copies final Anthropic citation wire fields once while collect ]; const wire = events.map(event => eventFrame(anthropicMessagesEventToSsePayload(event))); const stream = recorded(wire, true); - const rendered = renderStreamEvents('anthropic-messages', stream.events); - expect(JSON.parse(rendered[1]!.text)).toEqual(wire[1]!.event); - expect(JSON.parse(rendered[2]!.text)).toEqual(wire[2]!.event); - const copied = streamEventsCopyText('anthropic-messages', stream.events); - expect(copied).toContain('"source":"https://example.com/source"'); - expect(copied).not.toContain('"url"'); expect(await collectStream('anthropic-messages', stream)).toMatchObject({ result: { content: [{ type: 'text', text: 'answer', citations: [citation, citation] }] }, error: null, truncated: false }); }); diff --git a/apps/web/src/components/requests/events.tsx b/apps/web/src/components/requests/events.tsx index fd54317fed..c4fc4d8d97 100644 --- a/apps/web/src/components/requests/events.tsx +++ b/apps/web/src/components/requests/events.tsx @@ -2,7 +2,6 @@ import { ChevronDownRegular, ChevronRightRegular, DismissRegular, MoreHorizontal import { useCallback, useMemo, useRef, useState, type ReactNode } from 'react'; import { List, useDynamicRowHeight, type ListImperativeAPI, type RowComponentProps } from 'react-window'; -import { renderStreamEvents, streamEventsCopyText, type CollectKind, type RenderedStreamEvent } from './stream-render'; import { fluentComponents } from '../../fluent'; import { useTranslation } from '../../i18n/translation'; import { CodeBlock } from '../ui/code-block'; @@ -12,11 +11,15 @@ import { PANEL_BAND_CLASS } from '../ui/panel'; import { useScrollAreaHost } from '../ui/scroll-area'; import { TooltipIconButton } from '../ui/tooltip-icon-button'; import { copyOutcomeIcon, useCopyLabel, useCopyToClipboard } from '../ui/use-copy-to-clipboard'; -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; const { Button, Menu, MenuItem, MenuItemCheckbox, MenuList, MenuPopover, MenuTrigger, Text, mergeClasses } = fluentComponents; -export type RenderedEvent = Omit & { timestamp?: number }; +export interface RenderedEvent { + event: string | null; + text: string; + parseError: string | null; + timestamp?: number; +} type IndexedEvent = RenderedEvent & { index: number }; interface EventRowProps { events: IndexedEvent[]; @@ -38,7 +41,7 @@ function EventRow({ index, style, ariaAttributes, events, collapsed, toggle, wra wrap={wrap} copyOutcome={outcomeFor()} onCopy={() => copy(event.text)} - language={event.parseError || event.text === '[DONE]' ? 'plain' : 'json'} + language={event.parseError ? 'plain' : 'json'} header={
{event.timestamp !== undefined && +{event.timestamp.toFixed(event.timestamp < 1 ? 3 : 0)}ms} @@ -48,12 +51,6 @@ function EventRow({ index, style, ariaAttributes, events, collapsed, toggle, wra
; } -export function EventList({ events, kind, toolbarStart }: { events: DumpStreamEvent[]; kind: CollectKind | null; toolbarStart: ReactNode }) { - const rendered = useMemo(() => renderStreamEvents(kind, events), [events, kind]); - const copyText = useMemo(() => streamEventsCopyText(kind, events), [events, kind]); - return ; -} - export function RenderedEventList({ events, copyText, toolbarStart, emptyText }: { events: RenderedEvent[]; copyText: string; toolbarStart: ReactNode; emptyText?: string }) { const { t } = useTranslation(); const [query, setQuery] = useState(''); diff --git a/apps/web/src/components/requests/stream-render.ts b/apps/web/src/components/requests/stream-render.ts index f1114e360d..88cb174057 100644 --- a/apps/web/src/components/requests/stream-render.ts +++ b/apps/web/src/components/requests/stream-render.ts @@ -2,25 +2,17 @@ import type { RecordedClientStream } from './run-stream'; import { errorMessage } from '../../lib/error-message'; import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; import { collectAnthropicMessagesProtocolEventsToResult } from '@floway-dev/protocols/anthropic-messages'; -import { sseFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { collectGeminiGenerateContentProtocolEventsToResult, - geminiGenerateContentProtocolFrameToSSEFrame, type GeminiGenerateContentStreamEvent, } from '@floway-dev/protocols/gemini-generate-content'; +import { collectOpenAIChatCompletionsProtocolEventsToResult } from '@floway-dev/protocols/openai-chat-completions'; import { - openaiChatCompletionsProtocolFrameToSSEFrame, - collectOpenAIChatCompletionsProtocolEventsToResult, -} from '@floway-dev/protocols/openai-chat-completions'; -import { - openaiCompletionsProtocolFrameToSSEFrame, reassembleOpenAICompletionsEvents, type OpenAICompletionsStreamEvent, } from '@floway-dev/protocols/openai-completions'; -import { - collectOpenAIResponsesProtocolEventsToResult, - openaiResponsesProtocolFrameToSSEFrame, -} from '@floway-dev/protocols/openai-responses'; +import { collectOpenAIResponsesProtocolEventsToResult } from '@floway-dev/protocols/openai-responses'; export type CollectKind = 'openai-completions' | 'openai-chat-completions' | 'anthropic-messages' | 'openai-responses' | 'gemini-generate-content'; @@ -30,13 +22,6 @@ export interface CollectedStream { truncated: boolean; } -export interface RenderedStreamEvent { - event: string | null; - text: string; - parseError: string | null; - timestamp: number; -} - export const detectCollectKind = (path: string): CollectKind | null => { if (path.includes('/messages')) return 'anthropic-messages'; if (path.includes('/responses')) return 'openai-responses'; @@ -73,42 +58,6 @@ export const collectStream = async (kind: CollectKind, { events, ended }: Record } }; -export const renderStreamEvents = (kind: CollectKind | null, events: readonly DumpStreamEvent[]): RenderedStreamEvent[] => { - return events.map(({ frame, ts }) => { - const sse = frameToSse(kind, frame); - if (!sse) return { event: null, text: '', parseError: null, timestamp: ts }; - if (frame.type === 'done') return { event: sse.event ?? '[DONE]', text: sse.data, parseError: null, timestamp: ts }; - try { - return { event: sse.event ?? null, text: JSON.stringify(JSON.parse(sse.data) as unknown, null, 2), parseError: null, timestamp: ts }; - } catch (error) { - return { event: sse.event ?? null, text: sse.data, parseError: errorMessage(error), timestamp: ts }; - } - }); -}; - -export const streamEventsCopyText = (kind: CollectKind | null, events: readonly DumpStreamEvent[]): string => { - return events.map(({ frame }) => { - const sse = frameToSse(kind, frame); - return sse ? `${sse.event ? `event: ${sse.event}\n` : ''}data: ${sse.data}\n` : ''; - }).filter(Boolean).join('\n'); -}; - async function* frames(events: readonly DumpStreamEvent[]) { for (const event of events) yield event.frame; } - -const frameToSse = (kind: CollectKind | null, frame: ProtocolFrame): SseFrame | null => { - try { - switch (kind) { - case 'openai-chat-completions': return openaiChatCompletionsProtocolFrameToSSEFrame(frame as never, { includeUsageChunk: true }); - case 'openai-completions': return openaiCompletionsProtocolFrameToSSEFrame(frame as never); - // The client stream already carries Anthropic's citation wire spelling. - case 'anthropic-messages': return frame.type === 'done' ? null : sseFrame(JSON.stringify(frame.event), (frame.event as { type: string }).type); - case 'openai-responses': return openaiResponsesProtocolFrameToSSEFrame(frame as never); - case 'gemini-generate-content': return geminiGenerateContentProtocolFrameToSSEFrame(frame as never); - default: return null; - } - } catch (error) { - return { type: 'sse', event: 'serialize_error', data: errorMessage(error) }; - } -}; diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index c5edcd5240..ddecbb410c 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -17,7 +17,7 @@ import type { import { gunzipBytes, gzipBytes } from '../shared/gzip.ts'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; -// Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.{run}.gz`. +// Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.run.gz`. // The hour segment remains useful for operator inspection; lifecycle and // collection are driven by the shared spilled_files registry. @@ -30,9 +30,6 @@ interface DumpRow { upstream_kind: string | null; upstream_hue: number | null; meta_json: string; - request_headers_json: string; - response_headers_json: string | null; - request_body_descriptor: string | null; response_body_descriptor: string | null; } @@ -136,7 +133,7 @@ export class FileDumpStore implements DumpStore { async get(keyId: string, recordId: DumpRecordId): Promise { const row = await this.db.prepare( 'SELECT d.id, d.upstream_id, u.name AS upstream_name, u.provider AS upstream_kind, u.hue AS upstream_hue, ' - + 'd.meta_json, d.request_headers_json, d.response_headers_json, d.request_body_descriptor, d.response_body_descriptor ' + + 'd.meta_json, d.response_body_descriptor ' + 'FROM dump_records d LEFT JOIN upstreams u ON u.id = d.upstream_id ' + 'JOIN api_keys k ON k.id = d.key_id AND k.deleted_at IS NULL AND k.dump_retention_seconds IS NOT NULL ' + 'WHERE d.key_id = ? AND d.id = ? AND d.created_at >= ? - k.dump_retention_seconds * 1000', From 5ddf5b30509fe927158e662da841979f927a7efb Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:48:46 +0800 Subject: [PATCH 152/156] refactor: collect recorded protocol frames directly Remove the retired edge-event DTO and synthetic timestamps from run decoding. Client collection consumes literal frames with their own recorded end marker, preserving EOF, truncation and citation behavior. --- .../components/requests/run-stream_test.ts | 10 +++++----- .../components/requests/stream-render_test.ts | 4 ++-- apps/web/src/components/requests/run-stream.ts | 12 ++++++------ .../src/components/requests/stream-render.ts | 17 ++++++++--------- packages/gateway/src/dump/schemas.ts | 10 ---------- packages/gateway/src/dump/types.ts | 5 ----- 6 files changed, 21 insertions(+), 37 deletions(-) diff --git a/apps/web/__tests__/components/requests/run-stream_test.ts b/apps/web/__tests__/components/requests/run-stream_test.ts index 29bf128ebd..7d95ad3998 100644 --- a/apps/web/__tests__/components/requests/run-stream_test.ts +++ b/apps/web/__tests__/components/requests/run-stream_test.ts @@ -19,9 +19,9 @@ for (const collected of [true, false]) { ...collected ? [selected] : [], ])); expect(clientStreamOf(events)).toEqual({ - events: [ - { ts: 0, frame: { type: 'event', event: { text: 'client' } } }, - { ts: 0, frame: { type: 'done' } }, + frames: [ + { type: 'event', event: { text: 'client' } }, + { type: 'done' }, ], ended, }); @@ -34,7 +34,7 @@ test('keeps a selected empty stream and distinguishes it from no client stream', { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': streamFact(1) } }, { type: 'stream.end', streamId: 1 }, - ])))).toEqual({ events: [], ended: true }); + ])))).toEqual({ frames: [], ended: true }); expect(clientStreamOf(toNdjson(encodeRun([ { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': null } }, @@ -49,7 +49,7 @@ test('selects the rendered classic Completions stream and its own end proof', () { type: 'stream.end', streamId: 2 }, { type: 'stage.leaved', stageId: 1, facts: { 'response.openaiCompletions.rendered': streamFact(2) } }, ])); - expect(clientStreamOf(events)).toMatchObject({ events: [{ frame: { event: { choices: [{ text: 'client' }] } } }, { frame: { type: 'done' } }], ended: true }); + expect(clientStreamOf(events)).toMatchObject({ frames: [{ event: { choices: [{ text: 'client' }] } }, { type: 'done' }], ended: true }); }); test('does not select a JSON Completions extension that resembles a decoded stream reference', () => { diff --git a/apps/web/__tests__/components/requests/stream-render_test.ts b/apps/web/__tests__/components/requests/stream-render_test.ts index f5f92d6439..ad64b59fa1 100644 --- a/apps/web/__tests__/components/requests/stream-render_test.ts +++ b/apps/web/__tests__/components/requests/stream-render_test.ts @@ -22,8 +22,8 @@ test('collects normal Chat Completions EOF without inventing a done frame', asyn const stream = recorded([chunk], true); const collected = await collectStream('openai-chat-completions', stream); expect(collected).toMatchObject({ result: { choices: [{ message: { content: 'hello' } }] }, error: null, truncated: false }); - expect(stream.events).toHaveLength(1); - expect(stream.events.map(({ frame }) => frame.type)).toEqual(['event']); + expect(stream.frames).toHaveLength(1); + expect(stream.frames.map(frame => frame.type)).toEqual(['event']); }); for (const terminal of [false, true]) { diff --git a/apps/web/src/components/requests/run-stream.ts b/apps/web/src/components/requests/run-stream.ts index 512555de8d..d20a1d7f98 100644 --- a/apps/web/src/components/requests/run-stream.ts +++ b/apps/web/src/components/requests/run-stream.ts @@ -1,8 +1,8 @@ -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; export interface RecordedClientStream { - readonly events: readonly DumpStreamEvent[]; + readonly frames: readonly ProtocolFrame[]; readonly ended: boolean; } @@ -10,7 +10,7 @@ export interface RecordedClientStream { // streaming responses consume them afterwards. Resolve the complete record before selecting. export const clientStreamOf = (ndjson: string): RecordedClientStream | null => { const read = createRunReader(); - const streams = new Map(); + const streams = new Map[]; ended: boolean }>(); let client: number | null = null; for (const line of ndjson.split('\n')) { if (line.length === 0) continue; @@ -24,11 +24,11 @@ export const clientStreamOf = (ndjson: string): RecordedClientStream | null => { } } if (event.type === 'stream.frame' || event.type === 'stream.end') { - const stream = streams.get(event.streamId) ?? { events: [], ended: false }; + const stream = streams.get(event.streamId) ?? { frames: [], ended: false }; if (event.type === 'stream.end') stream.ended = true; - else for (const frame of decoded?.frames ?? []) stream.events.push({ frame: frame as DumpStreamEvent['frame'], ts: 0 }); + else for (const frame of decoded?.frames ?? []) stream.frames.push(frame as ProtocolFrame); streams.set(event.streamId, stream); } } - return client === null ? null : streams.get(client) ?? { events: [], ended: false }; + return client === null ? null : streams.get(client) ?? { frames: [], ended: false }; }; diff --git a/apps/web/src/components/requests/stream-render.ts b/apps/web/src/components/requests/stream-render.ts index 88cb174057..a88f4f8e5f 100644 --- a/apps/web/src/components/requests/stream-render.ts +++ b/apps/web/src/components/requests/stream-render.ts @@ -1,6 +1,5 @@ import type { RecordedClientStream } from './run-stream'; import { errorMessage } from '../../lib/error-message'; -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; import { collectAnthropicMessagesProtocolEventsToResult } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { @@ -31,21 +30,21 @@ export const detectCollectKind = (path: string): CollectKind | null => { return null; }; -export const collectStream = async (kind: CollectKind, { events, ended }: RecordedClientStream): Promise => { +export const collectStream = async (kind: CollectKind, { frames, ended }: RecordedClientStream): Promise => { const complete = (result: unknown): CollectedStream => ({ result, error: null, truncated: !ended }); try { switch (kind) { case 'openai-chat-completions': - return complete(await collectOpenAIChatCompletionsProtocolEventsToResult(frames(events) as never)); + return complete(await collectOpenAIChatCompletionsProtocolEventsToResult(asIterable(frames) as never)); case 'anthropic-messages': - return complete(await collectAnthropicMessagesProtocolEventsToResult(frames(events) as never)); + return complete(await collectAnthropicMessagesProtocolEventsToResult(asIterable(frames) as never)); case 'openai-responses': - return complete(await collectOpenAIResponsesProtocolEventsToResult(frames(events) as never)); + return complete(await collectOpenAIResponsesProtocolEventsToResult(asIterable(frames) as never)); case 'gemini-generate-content': - return complete(await collectGeminiGenerateContentProtocolEventsToResult(frames(events) as AsyncIterable>)); + return complete(await collectGeminiGenerateContentProtocolEventsToResult(asIterable(frames) as AsyncIterable>)); case 'openai-completions': { const stream = (async function* () { - for (const { frame } of events) { + for (const frame of frames) { const typed = frame as ProtocolFrame; if (typed.type === 'event') yield typed.event; } @@ -58,6 +57,6 @@ export const collectStream = async (kind: CollectKind, { events, ended }: Record } }; -async function* frames(events: readonly DumpStreamEvent[]) { - for (const event of events) yield event.frame; +async function* asIterable(frames: readonly ProtocolFrame[]) { + yield* frames; } diff --git a/packages/gateway/src/dump/schemas.ts b/packages/gateway/src/dump/schemas.ts index 6944397b17..55624941e7 100644 --- a/packages/gateway/src/dump/schemas.ts +++ b/packages/gateway/src/dump/schemas.ts @@ -46,16 +46,6 @@ export const dumpBodyDescriptorSchema = z.object({ type: z.literal('run'), }).strict(); -const dumpProtocolFrameSchema = z.discriminatedUnion('type', [ - z.object({ type: z.literal('event'), event: z.unknown() }).strict(), - z.object({ type: z.literal('done') }).strict(), -]); - -export const dumpStreamEventSchema = z.object({ - frame: dumpProtocolFrameSchema, - ts: z.number(), -}).strict(); - export const dumpBrokerFrameSchema = z.object({ event: z.literal('appended'), data: dumpMetadataSchema, diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index 91a2fd572b..94126078be 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -6,7 +6,6 @@ import type { z } from 'zod'; import type { dumpErrorSchema, dumpMetadataSchema, - dumpStreamEventSchema, dumpUpstreamRefSchema, } from './schemas.ts'; import type { DumpEvent } from '@floway-dev/pipeline'; @@ -30,10 +29,6 @@ export type DumpErrorMeta = z.infer; export type DumpMetadata = z.infer; -// A protocol frame decoded from a recorded stream. Its producing stage owns -// the event shape; client streams already contain their final wire projection. -export type DumpStreamEvent = z.infer; - // The completed NDJSON artifact is stored after owned readers and deferred work // settle. Cold detail reads rehydrate the same event bytes. export type StoredDumpRecord = { From e797eeb3db8310da243e03d544f692a4bcf9a753 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 10:07:35 +0800 Subject: [PATCH 153/156] Track hosted image emission through settlement and release Own the emitted lifecycle and expose its deferred exception outcome. Join delayed emission with native or already measured JSON usage so formatter faults retain quantities once and propagate the original error through drain, scoped diagnostics and status 500 metadata. Preserve neutral undemanded images and the existing upstream-abort verdict. --- .../hosted-tools/image-sub-request_test.ts | 66 +++++++++++++++++++ .../hosted-tools/image-sub-request.ts | 4 +- .../hosted-tools/image-sub-request/emit.ts | 66 +++++++++++++++---- .../hosted-tools/image-sub-request/facts.ts | 6 +- 4 files changed, 126 insertions(+), 16 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts index ad5f46cfba..750472bd36 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts @@ -2,6 +2,7 @@ import { afterEach, expect, test, vi } from 'vitest'; import * as imageRequest from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts'; import type { ImageGenerationRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts'; +import * as imageResult from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts'; import { runImageGenerationSubRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts'; import * as resolution from '../../../../../src/data-plane/providers/resolution.ts'; import { initDumpBroker, initDumpStore } from '../../../../../src/dump/registry.ts'; @@ -77,6 +78,9 @@ for (const action of ['generate', 'edit'] as const) for (const consume of [false else expect(frames).toEqual([]); const observations = await repo.usage.listAll(); expect(observations).toMatchObject([{ model: 'image-model', modelKey: 'backend-image', requests: 1 }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, neutral: 1, errorsNoOutput: 0, errorsWithOutput: 0 }]); + expect(record.meta.status).toBe(200); + expect(record.meta.error).toBeNull(); }); } @@ -101,3 +105,65 @@ test('hosted image preparation preserves an internal TypeError and records the f expect(entered).toBeDefined(); expect(events).toContainEqual(expect.objectContaining({ type: 'stage.failed', stageId: entered!.stageId })); }); + +const openGeneration = async (stream: boolean, signal?: AbortSignal) => { + const repo = new InMemoryRepo(); + initRepo(repo); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground, ...(signal === undefined ? {} : { abortSignal: signal }) }); + const body = stream ? new Response([ + { type: 'image_generation.partial_image', partial_image_index: 0, b64_json: 'preview' }, + { type: 'image_generation.completed', b64_json: 'pixels', usage: { input_tokens: 7, output_tokens: 3 } }, + ].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } }) + : Response.json({ data: [{ b64_json: 'pixels' }], usage: { input_tokens: 7, output_tokens: 3 } }); + const dial = vi.fn(async () => ({ response: body, modelKey: 'backend-image' })); + const plain = stubModelCandidate({ model: { id: 'image-model', kind: 'image', endpoints: { openaiImagesGenerations: {} } } }); + const candidate = { ...plain, provider: { ...plain.provider, pipelines: { openaiImagesGenerations: stubProviderPipeline('openaiImagesGenerations', dial) } } }; + vi.spyOn(resolution, 'enumerateModelCandidates').mockResolvedValue({ candidates: [candidate], sawModel: true, failedUpstreams: [] }); + const call = await runImageGenerationSubRequest(parent, { prompt: 'tree', action: 'generate', config: { model: 'image-model', action: 'generate', ...(stream ? { partial_images: 1 } : {}) }, sources: [] }); + return { repo, dumps, dump, call, dial }; +}; + +for (const stream of [false, true]) test(`a ${stream ? 'streamed' : 'JSON'} child emission exception keeps measured usage and fails through drain`, async () => { + const { repo, dumps, dump, call, dial } = await openGeneration(stream); + const fault = new TypeError('image lifecycle formatter broke', { cause: new Error('invalid rendering state') }); + vi.spyOn(imageResult, 'imageTerminal').mockImplementation(() => { throw fault; }); + if (stream) expect(await call.lifecycle.next()).toMatchObject({ done: false }); + await expect(call.lifecycle.next()).rejects.toBe(fault); + await expect(call.drain()).rejects.toBe(fault); + dump?.finalize(200, 0); + await flushBackground(); + expect(dial).toHaveBeenCalledOnce(); + expect(await repo.usage.listAll()).toMatchObject([{ + requests: 1, metrics: [ + { metric: 'input_tokens', quantity: '7' }, { metric: 'output_tokens', quantity: '3' }, + ], + }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, neutral: 0, errorsNoOutput: 1 }]); + const record = dumps.stored.find(item => item.record.meta.path === '/images/generations')!.record; + expect(record.meta).toMatchObject({ status: 500, inputTokens: 7, outputTokens: 3, error: { kind: 'failed', reason: fault.message } }); + const read = createRunReader(); + const logs: unknown[] = []; + for (const item of eventsOf(record)) { + const event = item as unknown as DumpEvent; + read(event); + if (event.type === 'stage.log' && event.fields !== undefined) logs.push(read.decode(event.fields)); + } + expect(logs).toContainEqual(expect.objectContaining({ error: expect.objectContaining({ error: expect.objectContaining({ message: fault.message, cause: expect.objectContaining({ error: expect.objectContaining({ message: 'invalid rendering state' }) }) }) }) })); +}); + +test('an actual parent abort keeps the source failure verdict and original streamed HTTP status', async () => { + const controller = new AbortController(); + const { repo, dumps, dump, call } = await openGeneration(true, controller.signal); + controller.abort(new Error('parent canceled')); + expect(controller.signal.aborted).toBe(true); + await call.lifecycle.return(undefined as never); + await expect(call.drain()).rejects.toThrow('OpenAI Images stream ended without a completed event.'); + dump?.finalize(200, 0); + await flushBackground(); + expect(await repo.usage.listAll()).toMatchObject([{ requests: 1, metrics: [] }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, errorsNoOutput: 1, neutral: 0 }]); + const record = dumps.stored.find(item => item.record.meta.path === '/images/generations')!.record; + expect(record.meta).toMatchObject({ status: 200, error: { kind: 'failed', reason: 'OpenAI Images stream ended without a completed event.' } }); +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts index 20e38e9254..1617468b4d 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts @@ -42,15 +42,17 @@ export const runImageGenerationSubRequest = async ( return { lifecycle: facts['response.imageGeneration.lifecycle'], drain: async () => { + let status = facts['response.http.status']; try { // Releasing the native body also drives an unread image to its usage terminal. // Waiting for that reading first would prevent the owner from ever starting it. await drain(); } catch (error) { dump?.failed(error); + if ((await facts['response.imageGeneration.lifecycleOutcome']).kind === 'exception') status = 500; throw error; } finally { - dump?.finalize(facts['response.http.status'], 0); + dump?.finalize(status, 0); } }, }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts index 1e044d5daa..32d4027598 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts @@ -1,26 +1,26 @@ -import type { Fields } from './facts.ts'; +import type { Fields, ImageLifecycleOutcome } from './facts.ts'; import { errorFromBody, extractEcho, imageTerminal, isRetryableImageError, projectImageStreamEvent, serverError, type ImageOutcome } from './result.ts'; -import type { RunDump } from '../../../../../dump/run-sink.ts'; +import { streamReferenceOf, type RunDump } from '../../../../../dump/run-sink.ts'; import { isFailure } from '../../../../pipeline/facts.ts'; import type { GatewayServices } from '../../../../pipeline/services.ts'; import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../types.ts'; -import { defineStage, move } from '@floway-dev/pipeline'; +import { defineStage, move, own, defer } from '@floway-dev/pipeline'; import { eventFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIImagesResponse } from '@floway-dev/protocols/openai-images'; export const emitHostedImageGeneration = defineStage< Fields<'request.imageGeneration.canonical'>, Fields<'request.imageGeneration.canonical'>, - Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status'>, - Fields<'response.imageGeneration.lifecycle' | 'response.openaiImages.streamedUsage' | 'response.http.status'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.usage.billable'>, + Fields<'response.imageGeneration.lifecycle' | 'response.imageGeneration.lifecycleOutcome' | 'response.openaiImages.streamedUsage' | 'response.http.status'>, GatewayServices >({ name: 'emitHostedImageGeneration', through: { request: { needs: ['request.imageGeneration.canonical'], consumes: [], provides: [] }, response: { - needs: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status'], - consumes: ['response.openaiImages.canonical'], provides: ['response.imageGeneration.lifecycle'], + needs: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status', 'response.usage.billable'], + consumes: ['response.openaiImages.canonical'], provides: ['response.imageGeneration.lifecycle', 'response.imageGeneration.lifecycleOutcome', 'response.openaiImages.streamedUsage'], }, }, execute: async (facts, next, use) => { @@ -35,9 +35,20 @@ export const emitHostedImageGeneration = defineStage< : errorFromBody(JSON.stringify(answer.body), answer.status); outcome = { ok: false, error: { ...error, type: error.type ?? 'image_generation_error', retryable: isRetryableImageError(error.code, error.type) } }; } else if (Symbol.asyncIterator in answer) { + const reader = answer[Symbol.asyncIterator](); try { - for await (const event of answer) { - const signal = projectImageStreamEvent(event); + for (;;) { + let step; + try { step = await reader.next(); } catch (error) { + use.gateway.dump?.failed(error); + outcome = { ok: false, error: serverError(error) }; + break; + } + if (step.done) { + outcome = { ok: false, error: serverError(new Error('Image backend stream ended without a completed image.')) }; + break; + } + const signal = projectImageStreamEvent(step.value); if (signal === null) continue; if (signal.kind === 'partial') { yield { type: 'response.image_generation_call.partial_image', partial_image_index: signal.index, partial_image_b64: signal.b64, ...signal.echo }; @@ -49,17 +60,44 @@ export const emitHostedImageGeneration = defineStage< return imageTerminal(request.prompt, request.action, { ok: false, error: signal.error }); } } - outcome = { ok: false, error: serverError(new Error('Image backend stream ended without a completed image.')) }; - } catch (error) { - use.gateway.dump?.failed(error); - outcome = { ok: false, error: serverError(error) }; + } finally { + await reader.return?.(); } } else { outcome = imageOutcome(answer); } return imageTerminal(request.prompt, request.action, outcome); })(); - return { ...rest, 'response.imageGeneration.lifecycle': move(recordLifecycle(lifecycle, use.gateway.dump)) }; + // Formatting continues after the stage hands up. Releasing undemanded output stays + // neutral, while an observed emission exception must reach both settlement and drain. + const emission = Promise.withResolvers(); + let exception: Extract | null = null; + const recorded = recordLifecycle(lifecycle, use.gateway.dump); + const observed = (async function* () { + try { return yield* recorded; } catch (error) { + exception = move({ kind: 'exception' as const, error }); + use.gateway.dump?.failed(error); + await use.log.error('image lifecycle emission failed', { error }); + throw error; + } finally { + emission.resolve(exception ?? move({ kind: 'released' as const })); + } + })(); + const owned = own(Object.assign(observed, streamReferenceOf(recorded)), async () => { + try { await observed.return(undefined as never); } finally { + emission.resolve(exception ?? move({ kind: 'released' as const })); + } + if (exception !== null) throw exception.error; + }); + const reading = back['response.openaiImages.streamedUsage'] ?? Promise.resolve({ billable: back['response.usage.billable'], failed: false }); + return move({ + ...rest, + 'response.imageGeneration.lifecycle': owned, + 'response.imageGeneration.lifecycleOutcome': defer(emission.promise), + 'response.openaiImages.streamedUsage': defer(Promise.all([reading, emission.promise]).then(([outcome, completed]) => ({ + ...outcome, failed: outcome.failed || completed.kind === 'exception', + }))), + }); }, }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts index 6ca0482b0c..1c97eaa4e4 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts @@ -1,12 +1,16 @@ import type { ImageGenerationRequest } from './request.ts'; import type { OpenAIImagesFacts } from '../../../../openai-images/facts.ts'; import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../types.ts'; +import type { Deferred } from '@floway-dev/pipeline'; + +export type ImageLifecycleOutcome = { readonly kind: 'released' } | { readonly kind: 'exception'; readonly error: unknown }; export interface ImageSubRequestFacts extends OpenAIImagesFacts { 'request.imageGeneration.canonical': ImageGenerationRequest; + 'response.imageGeneration.lifecycleOutcome': Deferred; 'response.imageGeneration.lifecycle': AsyncGenerator; } export type Fields = { [P in K]: ImageSubRequestFacts[P] }; export type ImageSubRequestEntry = Fields<'request.imageGeneration.canonical' | 'ingress.http.headers'>; -export type ImageSubRequestExit = Fields<'response.imageGeneration.lifecycle' | 'response.openaiImages.streamedUsage' | 'response.http.status'>; +export type ImageSubRequestExit = Fields<'response.imageGeneration.lifecycle' | 'response.imageGeneration.lifecycleOutcome' | 'response.openaiImages.streamedUsage' | 'response.http.status'>; From bd94eebbd9f2e49eb5d58e011527673ba18ea510 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 10:36:50 +0800 Subject: [PATCH 154/156] refactor: remove the retired run view label --- apps/web/src/i18n/locales/en.ts | 1 - apps/web/src/i18n/locales/zh-Hans.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index 65f7026a6e..058a33ccfe 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -1052,7 +1052,6 @@ const en = { noKeys: 'No API key has request retention enabled', noKeysDescription: 'Enable request dump retention on an API key to start capturing requests.', apiKeysLink: 'API Keys', - run: 'Run', stages: 'Stages', requestFacts: 'Request facts', responseFacts: 'Response facts', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index b5d9416b6e..df935b347f 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -1004,7 +1004,6 @@ const zhHansCN = { noKeys: '没有 API 密钥启用请求保留', noKeysDescription: '在 API 密钥上启用请求转储保留后,即可开始捕获请求。', apiKeysLink: 'API 密钥', - run: '运行过程', stages: 'Stages', requestFacts: '请求 facts', responseFacts: '响应 facts', From 8282252a85d2cd8c5acd6936eae2969eb0fa32d5 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 10:38:03 +0800 Subject: [PATCH 155/156] fix: preserve finite search body read failures Give the finite Alpha Search reader ownership before it acquires the body lock, so cleanup preserves the original read failure. Exercise native body errors alongside transport failures and successful calls; retain actual call observations and independent child diagnostics. --- .../hosted-tools/search-sub-request_test.ts | 31 ++++++++++++++++--- .../search-sub-request/execute.ts | 4 +-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts index 2e8e019585..61f6adcc6a 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts @@ -20,14 +20,17 @@ const request: WebSearchRequest = { settings: { search_context_size: 'medium' }, input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'Find the documentation' }] }], }; -for (const failed of [false, true]) { - test(`web search records backend content before dispatch and closes its ${failed ? 'failed' : 'successful'} child run`, async () => { - initRepo(new InMemoryRepo()); +for (const outcome of ['success', 'transportFailure', 'bodyReadFailure'] as const) { + test(`web search records backend content before dispatch and closes its ${outcome} child run`, async () => { + const failed = outcome !== 'success'; + const repo = new InMemoryRepo(); + initRepo(repo); const dumps = installDumpStubs(initDumpStore, initDumpBroker); const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); - const fault = new Error('search transport failed', { cause: new Error('socket reset') }); + const fault = new Error(outcome === 'bodyReadFailure' ? 'search body read failed' : 'search transport failed', { cause: new Error('socket reset') }); let dispatched = false; + let replyProduced = false; const invocation = runWebSearchSubRequest(parent, request, { session: { pageCache: new Map(), getProvider: () => { throw new Error('local provider must not dispatch in alpha mode'); }, apiKeyId: apiKey.id }, alpha: { @@ -35,7 +38,11 @@ for (const failed of [false, true]) { dispatched = true; expect(body).toEqual({ id: 'search-session', commands: request.commands, settings: request.settings, input: request.input }); expect(Object.isFrozen(body.commands)).toBe(true); - if (failed) throw fault; + if (outcome === 'transportFailure') throw fault; + replyProduced = true; + if (outcome === 'bodyReadFailure') { + return new Response(new ReadableStream({ pull: controller => controller.error(fault) }), { headers: { 'content-type': 'application/json' } }); + } return new Response(JSON.stringify({ output: 'documentation' }), { headers: { 'content-type': 'application/json' } }); }), }, @@ -43,6 +50,7 @@ for (const failed of [false, true]) { if (failed) await expect(invocation).rejects.toBe(fault); else expect(await invocation).toMatchObject({ outputText: 'documentation' }); expect(dispatched).toBe(true); + expect(replyProduced).toBe(outcome !== 'transportFailure'); dump?.finalize(200, 0); await flushBackground(); expect(dumps.stored).toHaveLength(2); @@ -50,11 +58,24 @@ for (const failed of [false, true]) { const record = dumps.stored.find(item => item.record.meta.path !== '/v1/responses')!.record; expect(record.meta.path).toBe('/alpha/search'); expect(record.meta.status).toBe(failed ? 500 : 200); + expect(record.meta.error).toEqual(failed ? { kind: 'failed', reason: fault.message } : null); + const usage = await repo.usage.listAll(); + if (outcome === 'transportFailure') expect(usage).toEqual([]); + else { + expect(usage).toHaveLength(1); + expect(usage[0]).toMatchObject({ requests: 1, modelKey: 'search-model', metrics: [] }); + } const read = createRunReader(); const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); const entered = decoded.find(event => event?.facts && 'request.webSearch.canonical' in event.facts); expect(entered?.facts?.['request.webSearch.canonical']).toEqual(request); expect(decoded).toContainEqual(expect.objectContaining({ facts: expect.objectContaining({ 'request.provider.payload': { id: 'search-session', commands: request.commands, settings: request.settings, input: request.input } }) })); expect(new TextDecoder().decode(record.events)).not.toContain('searchCall'); + if (failed) { + const events = eventsOf(record); + const stage = events.find(event => event.type === 'stage.entered' && event.name === 'runWebSearchCall'); + expect(stage).toBeDefined(); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.failed', stageId: stage!.stageId })); + } }); } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts index d6c2d22a1a..0e14fb97aa 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts @@ -44,9 +44,9 @@ export const runWebSearchCall = defineStage< const exchange = back['response.http.exchange']; if (exchange.type === 'transportFailure') throw exchange.error; const response = exchangeResponse(exchange); - // Formatting reads the finite body before the child returns; release then has no bytes left. - const bytes = await response.arrayBuffer(); + // The finite reader owns completion, including rejection; canceling its locked body would mask the read error. spentBody(exchange.body); + const bytes = await response.arrayBuffer(); return new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]) }); }; const ir = await execute(facts['request.webSearch.canonical'], use.webSearch, dispatch); From f783095d4ca6e0bdf91ce33d502a452e4a7fda51 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 10:41:57 +0800 Subject: [PATCH 156/156] fix: preserve chat error body read failures Transfer finite error-body ownership before decoding, so resource cleanup cannot add a locked-stream failure. Verify the actual HTTP boundary preserves the original error and cause, records its failed verdict, and bills the called provider once. --- .../chat/openai-chat-completions/http_test.ts | 29 +++++++++++++++++++ .../data-plane/chat/shared/provider-answer.ts | 6 ++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts index 12d5f8b61b..87440ece2c 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts @@ -468,6 +468,35 @@ test('POST /v1/chat/completions leaves TTFT absent on a failure before output', assertEquals(meta.ttftMs, null); }); +test('actual HTTP error-body read failure preserves its original error and called observation', async () => { + const repo = installRepo(); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const fault = new Error('upstream error body read failed', { cause: new TypeError('socket reading failed') }); + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}); + const call = vi.fn(async (): Promise> => ({ + ok: false, modelKey: 'called-model', + response: new Response(new ReadableStream({ pull: controller => controller.error(fault) }), { status: 503, headers: { 'content-type': 'application/json' } }), + })); + queueCandidates([makeCandidate({ callOpenAIChatCompletions: call })]); + const response = await makeApp(undefined, { dumpRetentionSeconds: 3600 }).request('/v1/chat/completions', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'test-model', messages: [{ role: 'user', content: 'hello' }] }), + }); + expect(response.status).toBe(500); + expect(await response.json()).toMatchObject({ error: { name: fault.name, message: fault.message, stack: fault.stack, cause: { name: 'TypeError', message: 'socket reading failed' } } }); + await flushBackground(); + expect(call).toHaveBeenCalledOnce(); + expect(logged).toHaveBeenCalledOnce(); + expect(logged).toHaveBeenCalledWith(fault); + logged.mockRestore(); + expect(dumps.stored).toHaveLength(1); + expect(dumps.stored[0]!.record.meta).toMatchObject({ status: 500, error: { kind: 'failed', reason: fault.message } }); + const usage = await repo.usage.listAll(); + expect(usage).toHaveLength(1); + expect(usage[0]).toMatchObject({ modelKey: 'called-model', requests: 1, metrics: [] }); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, errorsNoOutput: 1, errorsWithOutput: 0 }]); +}); + test('actual HTTP cancellation before the first client frame releases the called provider and settles failure once', async () => { const repo = installRepo(); let unblock!: () => void; diff --git a/packages/gateway/src/data-plane/chat/shared/provider-answer.ts b/packages/gateway/src/data-plane/chat/shared/provider-answer.ts index 37c64d58e6..493fe16819 100644 --- a/packages/gateway/src/data-plane/chat/shared/provider-answer.ts +++ b/packages/gateway/src/data-plane/chat/shared/provider-answer.ts @@ -18,9 +18,11 @@ export const providerChatAnswer = async (candid billable, body: 'kind' in output && output.kind === 'stream' ? exchange.body : spentBody(exchange.body), }; - const body = await readUpstreamBody(exchangeResponse(exchange)); + const response = exchangeResponse(exchange); + spentBody(exchange.body); + const body = await readUpstreamBody(response); return { answer: { status: exchange.status, message: body.text, ...('json' in body ? { body: body.json } : {}) }, - status: exchange.status, headers: exchange.headers, billable, body: spentBody(exchange.body), + status: exchange.status, headers: exchange.headers, billable, body: exchange.body, }; };